1ee3a5b23SShameer Kolothum // SPDX-License-Identifier: GPL-2.0-only 2ee3a5b23SShameer Kolothum /* 3ee3a5b23SShameer Kolothum * Copyright (c) 2021, HiSilicon Ltd. 4ee3a5b23SShameer Kolothum */ 5ee3a5b23SShameer Kolothum 6ee3a5b23SShameer Kolothum #include <linux/device.h> 7ee3a5b23SShameer Kolothum #include <linux/eventfd.h> 8ee3a5b23SShameer Kolothum #include <linux/file.h> 9ee3a5b23SShameer Kolothum #include <linux/hisi_acc_qm.h> 10ee3a5b23SShameer Kolothum #include <linux/interrupt.h> 11ee3a5b23SShameer Kolothum #include <linux/module.h> 12ee3a5b23SShameer Kolothum #include <linux/pci.h> 13ee3a5b23SShameer Kolothum #include <linux/vfio.h> 14ee3a5b23SShameer Kolothum #include <linux/vfio_pci_core.h> 15b0eed085SLongfang Liu #include <linux/anon_inodes.h> 16b0eed085SLongfang Liu 17b0eed085SLongfang Liu #include "hisi_acc_vfio_pci.h" 18b0eed085SLongfang Liu 19b0eed085SLongfang Liu /* return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */ 20b0eed085SLongfang Liu static int qm_wait_dev_not_ready(struct hisi_qm *qm) 21b0eed085SLongfang Liu { 22b0eed085SLongfang Liu u32 val; 23b0eed085SLongfang Liu 24b0eed085SLongfang Liu return readl_relaxed_poll_timeout(qm->io_base + QM_VF_STATE, 25b0eed085SLongfang Liu val, !(val & 0x1), MB_POLL_PERIOD_US, 26b0eed085SLongfang Liu MB_POLL_TIMEOUT_US); 27b0eed085SLongfang Liu } 28b0eed085SLongfang Liu 29b0eed085SLongfang Liu /* 30b0eed085SLongfang Liu * Each state Reg is checked 100 times, 31b0eed085SLongfang Liu * with a delay of 100 microseconds after each check 32b0eed085SLongfang Liu */ 33b0eed085SLongfang Liu static u32 qm_check_reg_state(struct hisi_qm *qm, u32 regs) 34b0eed085SLongfang Liu { 35b0eed085SLongfang Liu int check_times = 0; 36b0eed085SLongfang Liu u32 state; 37b0eed085SLongfang Liu 38b0eed085SLongfang Liu state = readl(qm->io_base + regs); 39b0eed085SLongfang Liu while (state && check_times < ERROR_CHECK_TIMEOUT) { 40b0eed085SLongfang Liu udelay(CHECK_DELAY_TIME); 41b0eed085SLongfang Liu state = readl(qm->io_base + regs); 42b0eed085SLongfang Liu check_times++; 43b0eed085SLongfang Liu } 44b0eed085SLongfang Liu 45b0eed085SLongfang Liu return state; 46b0eed085SLongfang Liu } 47b0eed085SLongfang Liu 48b0eed085SLongfang Liu static int qm_read_regs(struct hisi_qm *qm, u32 reg_addr, 49b0eed085SLongfang Liu u32 *data, u8 nums) 50b0eed085SLongfang Liu { 51b0eed085SLongfang Liu int i; 52b0eed085SLongfang Liu 53b0eed085SLongfang Liu if (nums < 1 || nums > QM_REGS_MAX_LEN) 54b0eed085SLongfang Liu return -EINVAL; 55b0eed085SLongfang Liu 56b0eed085SLongfang Liu for (i = 0; i < nums; i++) { 57b0eed085SLongfang Liu data[i] = readl(qm->io_base + reg_addr); 58b0eed085SLongfang Liu reg_addr += QM_REG_ADDR_OFFSET; 59b0eed085SLongfang Liu } 60b0eed085SLongfang Liu 61b0eed085SLongfang Liu return 0; 62b0eed085SLongfang Liu } 63b0eed085SLongfang Liu 64b0eed085SLongfang Liu static int qm_write_regs(struct hisi_qm *qm, u32 reg, 65b0eed085SLongfang Liu u32 *data, u8 nums) 66b0eed085SLongfang Liu { 67b0eed085SLongfang Liu int i; 68b0eed085SLongfang Liu 69b0eed085SLongfang Liu if (nums < 1 || nums > QM_REGS_MAX_LEN) 70b0eed085SLongfang Liu return -EINVAL; 71b0eed085SLongfang Liu 72b0eed085SLongfang Liu for (i = 0; i < nums; i++) 73b0eed085SLongfang Liu writel(data[i], qm->io_base + reg + i * QM_REG_ADDR_OFFSET); 74b0eed085SLongfang Liu 75b0eed085SLongfang Liu return 0; 76b0eed085SLongfang Liu } 77b0eed085SLongfang Liu 78b0eed085SLongfang Liu static int qm_get_vft(struct hisi_qm *qm, u32 *base) 79b0eed085SLongfang Liu { 80b0eed085SLongfang Liu u64 sqc_vft; 81b0eed085SLongfang Liu u32 qp_num; 82b0eed085SLongfang Liu int ret; 83b0eed085SLongfang Liu 84b0eed085SLongfang Liu ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1); 85b0eed085SLongfang Liu if (ret) 86b0eed085SLongfang Liu return ret; 87b0eed085SLongfang Liu 88b0eed085SLongfang Liu sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | 89b0eed085SLongfang Liu ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 90b0eed085SLongfang Liu QM_XQC_ADDR_OFFSET); 91b0eed085SLongfang Liu *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); 92b0eed085SLongfang Liu qp_num = (QM_SQC_VFT_NUM_MASK_V2 & 93b0eed085SLongfang Liu (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; 94b0eed085SLongfang Liu 95b0eed085SLongfang Liu return qp_num; 96b0eed085SLongfang Liu } 97b0eed085SLongfang Liu 98b0eed085SLongfang Liu static int qm_get_sqc(struct hisi_qm *qm, u64 *addr) 99b0eed085SLongfang Liu { 100b0eed085SLongfang Liu int ret; 101b0eed085SLongfang Liu 102b0eed085SLongfang Liu ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, 0, 0, 1); 103b0eed085SLongfang Liu if (ret) 104b0eed085SLongfang Liu return ret; 105b0eed085SLongfang Liu 106b0eed085SLongfang Liu *addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | 107b0eed085SLongfang Liu ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 108b0eed085SLongfang Liu QM_XQC_ADDR_OFFSET); 109b0eed085SLongfang Liu 110b0eed085SLongfang Liu return 0; 111b0eed085SLongfang Liu } 112b0eed085SLongfang Liu 113b0eed085SLongfang Liu static int qm_get_cqc(struct hisi_qm *qm, u64 *addr) 114b0eed085SLongfang Liu { 115b0eed085SLongfang Liu int ret; 116b0eed085SLongfang Liu 117b0eed085SLongfang Liu ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, 0, 0, 1); 118b0eed085SLongfang Liu if (ret) 119b0eed085SLongfang Liu return ret; 120b0eed085SLongfang Liu 121b0eed085SLongfang Liu *addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | 122b0eed085SLongfang Liu ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 123b0eed085SLongfang Liu QM_XQC_ADDR_OFFSET); 124b0eed085SLongfang Liu 125b0eed085SLongfang Liu return 0; 126b0eed085SLongfang Liu } 127b0eed085SLongfang Liu 128b0eed085SLongfang Liu static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data) 129b0eed085SLongfang Liu { 130b0eed085SLongfang Liu struct device *dev = &qm->pdev->dev; 131b0eed085SLongfang Liu int ret; 132b0eed085SLongfang Liu 133b0eed085SLongfang Liu ret = qm_read_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1); 134b0eed085SLongfang Liu if (ret) { 135b0eed085SLongfang Liu dev_err(dev, "failed to read QM_VF_AEQ_INT_MASK\n"); 136b0eed085SLongfang Liu return ret; 137b0eed085SLongfang Liu } 138b0eed085SLongfang Liu 139b0eed085SLongfang Liu ret = qm_read_regs(qm, QM_VF_EQ_INT_MASK, &vf_data->eq_int_mask, 1); 140b0eed085SLongfang Liu if (ret) { 141b0eed085SLongfang Liu dev_err(dev, "failed to read QM_VF_EQ_INT_MASK\n"); 142b0eed085SLongfang Liu return ret; 143b0eed085SLongfang Liu } 144b0eed085SLongfang Liu 145b0eed085SLongfang Liu ret = qm_read_regs(qm, QM_IFC_INT_SOURCE_V, 146b0eed085SLongfang Liu &vf_data->ifc_int_source, 1); 147b0eed085SLongfang Liu if (ret) { 148b0eed085SLongfang Liu dev_err(dev, "failed to read QM_IFC_INT_SOURCE_V\n"); 149b0eed085SLongfang Liu return ret; 150b0eed085SLongfang Liu } 151b0eed085SLongfang Liu 152b0eed085SLongfang Liu ret = qm_read_regs(qm, QM_IFC_INT_MASK, &vf_data->ifc_int_mask, 1); 153b0eed085SLongfang Liu if (ret) { 154b0eed085SLongfang Liu dev_err(dev, "failed to read QM_IFC_INT_MASK\n"); 155b0eed085SLongfang Liu return ret; 156b0eed085SLongfang Liu } 157b0eed085SLongfang Liu 158b0eed085SLongfang Liu ret = qm_read_regs(qm, QM_IFC_INT_SET_V, &vf_data->ifc_int_set, 1); 159b0eed085SLongfang Liu if (ret) { 160b0eed085SLongfang Liu dev_err(dev, "failed to read QM_IFC_INT_SET_V\n"); 161b0eed085SLongfang Liu return ret; 162b0eed085SLongfang Liu } 163b0eed085SLongfang Liu 164b0eed085SLongfang Liu ret = qm_read_regs(qm, QM_PAGE_SIZE, &vf_data->page_size, 1); 165b0eed085SLongfang Liu if (ret) { 166b0eed085SLongfang Liu dev_err(dev, "failed to read QM_PAGE_SIZE\n"); 167b0eed085SLongfang Liu return ret; 168b0eed085SLongfang Liu } 169b0eed085SLongfang Liu 170b0eed085SLongfang Liu /* QM_EQC_DW has 7 regs */ 171b0eed085SLongfang Liu ret = qm_read_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7); 172b0eed085SLongfang Liu if (ret) { 173b0eed085SLongfang Liu dev_err(dev, "failed to read QM_EQC_DW\n"); 174b0eed085SLongfang Liu return ret; 175b0eed085SLongfang Liu } 176b0eed085SLongfang Liu 177b0eed085SLongfang Liu /* QM_AEQC_DW has 7 regs */ 178b0eed085SLongfang Liu ret = qm_read_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7); 179b0eed085SLongfang Liu if (ret) { 180b0eed085SLongfang Liu dev_err(dev, "failed to read QM_AEQC_DW\n"); 181b0eed085SLongfang Liu return ret; 182b0eed085SLongfang Liu } 183b0eed085SLongfang Liu 184b0eed085SLongfang Liu return 0; 185b0eed085SLongfang Liu } 186b0eed085SLongfang Liu 187b0eed085SLongfang Liu static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data) 188b0eed085SLongfang Liu { 189b0eed085SLongfang Liu struct device *dev = &qm->pdev->dev; 190b0eed085SLongfang Liu int ret; 191b0eed085SLongfang Liu 192b0eed085SLongfang Liu /* check VF state */ 193b0eed085SLongfang Liu if (unlikely(hisi_qm_wait_mb_ready(qm))) { 194b0eed085SLongfang Liu dev_err(&qm->pdev->dev, "QM device is not ready to write\n"); 195b0eed085SLongfang Liu return -EBUSY; 196b0eed085SLongfang Liu } 197b0eed085SLongfang Liu 198b0eed085SLongfang Liu ret = qm_write_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1); 199b0eed085SLongfang Liu if (ret) { 200b0eed085SLongfang Liu dev_err(dev, "failed to write QM_VF_AEQ_INT_MASK\n"); 201b0eed085SLongfang Liu return ret; 202b0eed085SLongfang Liu } 203b0eed085SLongfang Liu 204b0eed085SLongfang Liu ret = qm_write_regs(qm, QM_VF_EQ_INT_MASK, &vf_data->eq_int_mask, 1); 205b0eed085SLongfang Liu if (ret) { 206b0eed085SLongfang Liu dev_err(dev, "failed to write QM_VF_EQ_INT_MASK\n"); 207b0eed085SLongfang Liu return ret; 208b0eed085SLongfang Liu } 209b0eed085SLongfang Liu 210b0eed085SLongfang Liu ret = qm_write_regs(qm, QM_IFC_INT_SOURCE_V, 211b0eed085SLongfang Liu &vf_data->ifc_int_source, 1); 212b0eed085SLongfang Liu if (ret) { 213b0eed085SLongfang Liu dev_err(dev, "failed to write QM_IFC_INT_SOURCE_V\n"); 214b0eed085SLongfang Liu return ret; 215b0eed085SLongfang Liu } 216b0eed085SLongfang Liu 217b0eed085SLongfang Liu ret = qm_write_regs(qm, QM_IFC_INT_MASK, &vf_data->ifc_int_mask, 1); 218b0eed085SLongfang Liu if (ret) { 219b0eed085SLongfang Liu dev_err(dev, "failed to write QM_IFC_INT_MASK\n"); 220b0eed085SLongfang Liu return ret; 221b0eed085SLongfang Liu } 222b0eed085SLongfang Liu 223b0eed085SLongfang Liu ret = qm_write_regs(qm, QM_IFC_INT_SET_V, &vf_data->ifc_int_set, 1); 224b0eed085SLongfang Liu if (ret) { 225b0eed085SLongfang Liu dev_err(dev, "failed to write QM_IFC_INT_SET_V\n"); 226b0eed085SLongfang Liu return ret; 227b0eed085SLongfang Liu } 228b0eed085SLongfang Liu 229b0eed085SLongfang Liu ret = qm_write_regs(qm, QM_QUE_ISO_CFG_V, &vf_data->que_iso_cfg, 1); 230b0eed085SLongfang Liu if (ret) { 231b0eed085SLongfang Liu dev_err(dev, "failed to write QM_QUE_ISO_CFG_V\n"); 232b0eed085SLongfang Liu return ret; 233b0eed085SLongfang Liu } 234b0eed085SLongfang Liu 235b0eed085SLongfang Liu ret = qm_write_regs(qm, QM_PAGE_SIZE, &vf_data->page_size, 1); 236b0eed085SLongfang Liu if (ret) { 237b0eed085SLongfang Liu dev_err(dev, "failed to write QM_PAGE_SIZE\n"); 238b0eed085SLongfang Liu return ret; 239b0eed085SLongfang Liu } 240b0eed085SLongfang Liu 241b0eed085SLongfang Liu /* QM_EQC_DW has 7 regs */ 242b0eed085SLongfang Liu ret = qm_write_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7); 243b0eed085SLongfang Liu if (ret) { 244b0eed085SLongfang Liu dev_err(dev, "failed to write QM_EQC_DW\n"); 245b0eed085SLongfang Liu return ret; 246b0eed085SLongfang Liu } 247b0eed085SLongfang Liu 248b0eed085SLongfang Liu /* QM_AEQC_DW has 7 regs */ 249b0eed085SLongfang Liu ret = qm_write_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7); 250b0eed085SLongfang Liu if (ret) { 251b0eed085SLongfang Liu dev_err(dev, "failed to write QM_AEQC_DW\n"); 252b0eed085SLongfang Liu return ret; 253b0eed085SLongfang Liu } 254b0eed085SLongfang Liu 255b0eed085SLongfang Liu return 0; 256b0eed085SLongfang Liu } 257b0eed085SLongfang Liu 258b0eed085SLongfang Liu static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, 259b0eed085SLongfang Liu u16 index, u8 priority) 260b0eed085SLongfang Liu { 261b0eed085SLongfang Liu u64 doorbell; 262b0eed085SLongfang Liu u64 dbase; 263b0eed085SLongfang Liu u16 randata = 0; 264b0eed085SLongfang Liu 265b0eed085SLongfang Liu if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ) 266b0eed085SLongfang Liu dbase = QM_DOORBELL_SQ_CQ_BASE_V2; 267b0eed085SLongfang Liu else 268b0eed085SLongfang Liu dbase = QM_DOORBELL_EQ_AEQ_BASE_V2; 269b0eed085SLongfang Liu 270b0eed085SLongfang Liu doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) | 271b0eed085SLongfang Liu ((u64)randata << QM_DB_RAND_SHIFT_V2) | 272b0eed085SLongfang Liu ((u64)index << QM_DB_INDEX_SHIFT_V2) | 273b0eed085SLongfang Liu ((u64)priority << QM_DB_PRIORITY_SHIFT_V2); 274b0eed085SLongfang Liu 275b0eed085SLongfang Liu writeq(doorbell, qm->io_base + dbase); 276b0eed085SLongfang Liu } 277b0eed085SLongfang Liu 278b0eed085SLongfang Liu static int pf_qm_get_qp_num(struct hisi_qm *qm, int vf_id, u32 *rbase) 279b0eed085SLongfang Liu { 280b0eed085SLongfang Liu unsigned int val; 281b0eed085SLongfang Liu u64 sqc_vft; 282b0eed085SLongfang Liu u32 qp_num; 283b0eed085SLongfang Liu int ret; 284b0eed085SLongfang Liu 285b0eed085SLongfang Liu ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 286b0eed085SLongfang Liu val & BIT(0), MB_POLL_PERIOD_US, 287b0eed085SLongfang Liu MB_POLL_TIMEOUT_US); 288b0eed085SLongfang Liu if (ret) 289b0eed085SLongfang Liu return ret; 290b0eed085SLongfang Liu 291b0eed085SLongfang Liu writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR); 292b0eed085SLongfang Liu /* 0 mean SQC VFT */ 293b0eed085SLongfang Liu writel(0x0, qm->io_base + QM_VFT_CFG_TYPE); 294b0eed085SLongfang Liu writel(vf_id, qm->io_base + QM_VFT_CFG); 295b0eed085SLongfang Liu 296b0eed085SLongfang Liu writel(0x0, qm->io_base + QM_VFT_CFG_RDY); 297b0eed085SLongfang Liu writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); 298b0eed085SLongfang Liu 299b0eed085SLongfang Liu ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, 300b0eed085SLongfang Liu val & BIT(0), MB_POLL_PERIOD_US, 301b0eed085SLongfang Liu MB_POLL_TIMEOUT_US); 302b0eed085SLongfang Liu if (ret) 303b0eed085SLongfang Liu return ret; 304b0eed085SLongfang Liu 305b0eed085SLongfang Liu sqc_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) | 306b0eed085SLongfang Liu ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 307b0eed085SLongfang Liu QM_XQC_ADDR_OFFSET); 308b0eed085SLongfang Liu *rbase = QM_SQC_VFT_BASE_MASK_V2 & 309b0eed085SLongfang Liu (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); 310b0eed085SLongfang Liu qp_num = (QM_SQC_VFT_NUM_MASK_V2 & 311b0eed085SLongfang Liu (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; 312b0eed085SLongfang Liu 313b0eed085SLongfang Liu return qp_num; 314b0eed085SLongfang Liu } 315b0eed085SLongfang Liu 316b0eed085SLongfang Liu static void qm_dev_cmd_init(struct hisi_qm *qm) 317b0eed085SLongfang Liu { 318b0eed085SLongfang Liu /* Clear VF communication status registers. */ 319b0eed085SLongfang Liu writel(0x1, qm->io_base + QM_IFC_INT_SOURCE_V); 320b0eed085SLongfang Liu 321b0eed085SLongfang Liu /* Enable pf and vf communication. */ 322b0eed085SLongfang Liu writel(0x0, qm->io_base + QM_IFC_INT_MASK); 323b0eed085SLongfang Liu } 324b0eed085SLongfang Liu 325b0eed085SLongfang Liu static int vf_qm_cache_wb(struct hisi_qm *qm) 326b0eed085SLongfang Liu { 327b0eed085SLongfang Liu unsigned int val; 328b0eed085SLongfang Liu 329b0eed085SLongfang Liu writel(0x1, qm->io_base + QM_CACHE_WB_START); 330b0eed085SLongfang Liu if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, 331b0eed085SLongfang Liu val, val & BIT(0), MB_POLL_PERIOD_US, 332b0eed085SLongfang Liu MB_POLL_TIMEOUT_US)) { 333b0eed085SLongfang Liu dev_err(&qm->pdev->dev, "vf QM writeback sqc cache fail\n"); 334b0eed085SLongfang Liu return -EINVAL; 335b0eed085SLongfang Liu } 336b0eed085SLongfang Liu 337b0eed085SLongfang Liu return 0; 338b0eed085SLongfang Liu } 339b0eed085SLongfang Liu 340b0eed085SLongfang Liu static void vf_qm_fun_reset(struct hisi_acc_vf_core_device *hisi_acc_vdev, 341b0eed085SLongfang Liu struct hisi_qm *qm) 342b0eed085SLongfang Liu { 343b0eed085SLongfang Liu int i; 344b0eed085SLongfang Liu 345b0eed085SLongfang Liu for (i = 0; i < qm->qp_num; i++) 346b0eed085SLongfang Liu qm_db(qm, i, QM_DOORBELL_CMD_SQ, 0, 1); 347b0eed085SLongfang Liu } 348b0eed085SLongfang Liu 349b0eed085SLongfang Liu static int vf_qm_func_stop(struct hisi_qm *qm) 350b0eed085SLongfang Liu { 351b0eed085SLongfang Liu return hisi_qm_mb(qm, QM_MB_CMD_PAUSE_QM, 0, 0, 0); 352b0eed085SLongfang Liu } 353b0eed085SLongfang Liu 354b0eed085SLongfang Liu static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, 355b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf) 356b0eed085SLongfang Liu { 357b0eed085SLongfang Liu struct acc_vf_data *vf_data = &migf->vf_data; 358b0eed085SLongfang Liu struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; 359b0eed085SLongfang Liu struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm; 360b0eed085SLongfang Liu struct device *dev = &vf_qm->pdev->dev; 361b0eed085SLongfang Liu u32 que_iso_state; 362b0eed085SLongfang Liu int ret; 363b0eed085SLongfang Liu 364b0eed085SLongfang Liu if (migf->total_length < QM_MATCH_SIZE) 365b0eed085SLongfang Liu return -EINVAL; 366b0eed085SLongfang Liu 367b0eed085SLongfang Liu if (vf_data->acc_magic != ACC_DEV_MAGIC) { 368b0eed085SLongfang Liu dev_err(dev, "failed to match ACC_DEV_MAGIC\n"); 369b0eed085SLongfang Liu return -EINVAL; 370b0eed085SLongfang Liu } 371b0eed085SLongfang Liu 372b0eed085SLongfang Liu if (vf_data->dev_id != hisi_acc_vdev->vf_dev->device) { 373b0eed085SLongfang Liu dev_err(dev, "failed to match VF devices\n"); 374b0eed085SLongfang Liu return -EINVAL; 375b0eed085SLongfang Liu } 376b0eed085SLongfang Liu 377b0eed085SLongfang Liu /* vf qp num check */ 378b0eed085SLongfang Liu ret = qm_get_vft(vf_qm, &vf_qm->qp_base); 379b0eed085SLongfang Liu if (ret <= 0) { 380b0eed085SLongfang Liu dev_err(dev, "failed to get vft qp nums\n"); 381b0eed085SLongfang Liu return -EINVAL; 382b0eed085SLongfang Liu } 383b0eed085SLongfang Liu 384b0eed085SLongfang Liu if (ret != vf_data->qp_num) { 385b0eed085SLongfang Liu dev_err(dev, "failed to match VF qp num\n"); 386b0eed085SLongfang Liu return -EINVAL; 387b0eed085SLongfang Liu } 388b0eed085SLongfang Liu 389b0eed085SLongfang Liu vf_qm->qp_num = ret; 390b0eed085SLongfang Liu 391b0eed085SLongfang Liu /* vf isolation state check */ 392b0eed085SLongfang Liu ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &que_iso_state, 1); 393b0eed085SLongfang Liu if (ret) { 394b0eed085SLongfang Liu dev_err(dev, "failed to read QM_QUE_ISO_CFG_V\n"); 395b0eed085SLongfang Liu return ret; 396b0eed085SLongfang Liu } 397b0eed085SLongfang Liu 398b0eed085SLongfang Liu if (vf_data->que_iso_cfg != que_iso_state) { 399b0eed085SLongfang Liu dev_err(dev, "failed to match isolation state\n"); 400b0eed085SLongfang Liu return ret; 401b0eed085SLongfang Liu } 402b0eed085SLongfang Liu 403b0eed085SLongfang Liu ret = qm_write_regs(vf_qm, QM_VF_STATE, &vf_data->vf_qm_state, 1); 404b0eed085SLongfang Liu if (ret) { 405b0eed085SLongfang Liu dev_err(dev, "failed to write QM_VF_STATE\n"); 406b0eed085SLongfang Liu return ret; 407b0eed085SLongfang Liu } 408b0eed085SLongfang Liu 409b0eed085SLongfang Liu hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; 410b0eed085SLongfang Liu return 0; 411b0eed085SLongfang Liu } 412b0eed085SLongfang Liu 413b0eed085SLongfang Liu static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, 414b0eed085SLongfang Liu struct acc_vf_data *vf_data) 415b0eed085SLongfang Liu { 416b0eed085SLongfang Liu struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm; 417b0eed085SLongfang Liu struct device *dev = &pf_qm->pdev->dev; 418b0eed085SLongfang Liu int vf_id = hisi_acc_vdev->vf_id; 419b0eed085SLongfang Liu int ret; 420b0eed085SLongfang Liu 421b0eed085SLongfang Liu vf_data->acc_magic = ACC_DEV_MAGIC; 422b0eed085SLongfang Liu /* save device id */ 423b0eed085SLongfang Liu vf_data->dev_id = hisi_acc_vdev->vf_dev->device; 424b0eed085SLongfang Liu 425b0eed085SLongfang Liu /* vf qp num save from PF */ 426b0eed085SLongfang Liu ret = pf_qm_get_qp_num(pf_qm, vf_id, &vf_data->qp_base); 427b0eed085SLongfang Liu if (ret <= 0) { 428b0eed085SLongfang Liu dev_err(dev, "failed to get vft qp nums!\n"); 429b0eed085SLongfang Liu return -EINVAL; 430b0eed085SLongfang Liu } 431b0eed085SLongfang Liu 432b0eed085SLongfang Liu vf_data->qp_num = ret; 433b0eed085SLongfang Liu 434b0eed085SLongfang Liu /* VF isolation state save from PF */ 435b0eed085SLongfang Liu ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &vf_data->que_iso_cfg, 1); 436b0eed085SLongfang Liu if (ret) { 437b0eed085SLongfang Liu dev_err(dev, "failed to read QM_QUE_ISO_CFG_V!\n"); 438b0eed085SLongfang Liu return ret; 439b0eed085SLongfang Liu } 440b0eed085SLongfang Liu 441b0eed085SLongfang Liu return 0; 442b0eed085SLongfang Liu } 443b0eed085SLongfang Liu 444b0eed085SLongfang Liu static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, 445b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf) 446b0eed085SLongfang Liu { 447b0eed085SLongfang Liu struct hisi_qm *qm = &hisi_acc_vdev->vf_qm; 448b0eed085SLongfang Liu struct device *dev = &qm->pdev->dev; 449b0eed085SLongfang Liu struct acc_vf_data *vf_data = &migf->vf_data; 450b0eed085SLongfang Liu int ret; 451b0eed085SLongfang Liu 452b0eed085SLongfang Liu /* Return if only match data was transferred */ 453b0eed085SLongfang Liu if (migf->total_length == QM_MATCH_SIZE) 454b0eed085SLongfang Liu return 0; 455b0eed085SLongfang Liu 456b0eed085SLongfang Liu if (migf->total_length < sizeof(struct acc_vf_data)) 457b0eed085SLongfang Liu return -EINVAL; 458b0eed085SLongfang Liu 459b0eed085SLongfang Liu qm->eqe_dma = vf_data->eqe_dma; 460b0eed085SLongfang Liu qm->aeqe_dma = vf_data->aeqe_dma; 461b0eed085SLongfang Liu qm->sqc_dma = vf_data->sqc_dma; 462b0eed085SLongfang Liu qm->cqc_dma = vf_data->cqc_dma; 463b0eed085SLongfang Liu 464b0eed085SLongfang Liu qm->qp_base = vf_data->qp_base; 465b0eed085SLongfang Liu qm->qp_num = vf_data->qp_num; 466b0eed085SLongfang Liu 467b0eed085SLongfang Liu ret = qm_set_regs(qm, vf_data); 468b0eed085SLongfang Liu if (ret) { 469b0eed085SLongfang Liu dev_err(dev, "Set VF regs failed\n"); 470b0eed085SLongfang Liu return ret; 471b0eed085SLongfang Liu } 472b0eed085SLongfang Liu 473b0eed085SLongfang Liu ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0); 474b0eed085SLongfang Liu if (ret) { 475b0eed085SLongfang Liu dev_err(dev, "Set sqc failed\n"); 476b0eed085SLongfang Liu return ret; 477b0eed085SLongfang Liu } 478b0eed085SLongfang Liu 479b0eed085SLongfang Liu ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0); 480b0eed085SLongfang Liu if (ret) { 481b0eed085SLongfang Liu dev_err(dev, "Set cqc failed\n"); 482b0eed085SLongfang Liu return ret; 483b0eed085SLongfang Liu } 484b0eed085SLongfang Liu 485b0eed085SLongfang Liu qm_dev_cmd_init(qm); 486b0eed085SLongfang Liu return 0; 487b0eed085SLongfang Liu } 488b0eed085SLongfang Liu 489b0eed085SLongfang Liu static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev, 490b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf) 491b0eed085SLongfang Liu { 492b0eed085SLongfang Liu struct acc_vf_data *vf_data = &migf->vf_data; 493b0eed085SLongfang Liu struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; 494b0eed085SLongfang Liu struct device *dev = &vf_qm->pdev->dev; 495b0eed085SLongfang Liu int ret; 496b0eed085SLongfang Liu 497b0eed085SLongfang Liu ret = vf_qm_get_match_data(hisi_acc_vdev, vf_data); 498b0eed085SLongfang Liu if (ret) 499b0eed085SLongfang Liu return ret; 500b0eed085SLongfang Liu 501b0eed085SLongfang Liu if (unlikely(qm_wait_dev_not_ready(vf_qm))) { 502b0eed085SLongfang Liu /* Update state and return with match data */ 503b0eed085SLongfang Liu vf_data->vf_qm_state = QM_NOT_READY; 504b0eed085SLongfang Liu hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; 505b0eed085SLongfang Liu migf->total_length = QM_MATCH_SIZE; 506b0eed085SLongfang Liu return 0; 507b0eed085SLongfang Liu } 508b0eed085SLongfang Liu 509b0eed085SLongfang Liu vf_data->vf_qm_state = QM_READY; 510b0eed085SLongfang Liu hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; 511b0eed085SLongfang Liu 512b0eed085SLongfang Liu ret = vf_qm_cache_wb(vf_qm); 513b0eed085SLongfang Liu if (ret) { 514b0eed085SLongfang Liu dev_err(dev, "failed to writeback QM Cache!\n"); 515b0eed085SLongfang Liu return ret; 516b0eed085SLongfang Liu } 517b0eed085SLongfang Liu 518b0eed085SLongfang Liu ret = qm_get_regs(vf_qm, vf_data); 519b0eed085SLongfang Liu if (ret) 520b0eed085SLongfang Liu return -EINVAL; 521b0eed085SLongfang Liu 522b0eed085SLongfang Liu /* Every reg is 32 bit, the dma address is 64 bit. */ 523b0eed085SLongfang Liu vf_data->eqe_dma = vf_data->qm_eqc_dw[2]; 524b0eed085SLongfang Liu vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET; 525b0eed085SLongfang Liu vf_data->eqe_dma |= vf_data->qm_eqc_dw[1]; 526b0eed085SLongfang Liu vf_data->aeqe_dma = vf_data->qm_aeqc_dw[2]; 527b0eed085SLongfang Liu vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET; 528b0eed085SLongfang Liu vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[1]; 529b0eed085SLongfang Liu 530b0eed085SLongfang Liu /* Through SQC_BT/CQC_BT to get sqc and cqc address */ 531b0eed085SLongfang Liu ret = qm_get_sqc(vf_qm, &vf_data->sqc_dma); 532b0eed085SLongfang Liu if (ret) { 533b0eed085SLongfang Liu dev_err(dev, "failed to read SQC addr!\n"); 534b0eed085SLongfang Liu return -EINVAL; 535b0eed085SLongfang Liu } 536b0eed085SLongfang Liu 537b0eed085SLongfang Liu ret = qm_get_cqc(vf_qm, &vf_data->cqc_dma); 538b0eed085SLongfang Liu if (ret) { 539b0eed085SLongfang Liu dev_err(dev, "failed to read CQC addr!\n"); 540b0eed085SLongfang Liu return -EINVAL; 541b0eed085SLongfang Liu } 542b0eed085SLongfang Liu 543b0eed085SLongfang Liu migf->total_length = sizeof(struct acc_vf_data); 544b0eed085SLongfang Liu return 0; 545b0eed085SLongfang Liu } 546b0eed085SLongfang Liu 547*245898ebSShameer Kolothum static struct hisi_acc_vf_core_device *hisi_acc_drvdata(struct pci_dev *pdev) 548*245898ebSShameer Kolothum { 549*245898ebSShameer Kolothum struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev); 550*245898ebSShameer Kolothum 551*245898ebSShameer Kolothum return container_of(core_device, struct hisi_acc_vf_core_device, 552*245898ebSShameer Kolothum core_device); 553*245898ebSShameer Kolothum } 554*245898ebSShameer Kolothum 555b0eed085SLongfang Liu /* Check the PF's RAS state and Function INT state */ 556b0eed085SLongfang Liu static int 557b0eed085SLongfang Liu hisi_acc_check_int_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) 558b0eed085SLongfang Liu { 559b0eed085SLongfang Liu struct hisi_qm *vfqm = &hisi_acc_vdev->vf_qm; 560b0eed085SLongfang Liu struct hisi_qm *qm = hisi_acc_vdev->pf_qm; 561b0eed085SLongfang Liu struct pci_dev *vf_pdev = hisi_acc_vdev->vf_dev; 562b0eed085SLongfang Liu struct device *dev = &qm->pdev->dev; 563b0eed085SLongfang Liu u32 state; 564b0eed085SLongfang Liu 565b0eed085SLongfang Liu /* Check RAS state */ 566b0eed085SLongfang Liu state = qm_check_reg_state(qm, QM_ABNORMAL_INT_STATUS); 567b0eed085SLongfang Liu if (state) { 568b0eed085SLongfang Liu dev_err(dev, "failed to check QM RAS state!\n"); 569b0eed085SLongfang Liu return -EBUSY; 570b0eed085SLongfang Liu } 571b0eed085SLongfang Liu 572b0eed085SLongfang Liu /* Check Function Communication state between PF and VF */ 573b0eed085SLongfang Liu state = qm_check_reg_state(vfqm, QM_IFC_INT_STATUS); 574b0eed085SLongfang Liu if (state) { 575b0eed085SLongfang Liu dev_err(dev, "failed to check QM IFC INT state!\n"); 576b0eed085SLongfang Liu return -EBUSY; 577b0eed085SLongfang Liu } 578b0eed085SLongfang Liu state = qm_check_reg_state(vfqm, QM_IFC_INT_SET_V); 579b0eed085SLongfang Liu if (state) { 580b0eed085SLongfang Liu dev_err(dev, "failed to check QM IFC INT SET state!\n"); 581b0eed085SLongfang Liu return -EBUSY; 582b0eed085SLongfang Liu } 583b0eed085SLongfang Liu 584b0eed085SLongfang Liu /* Check submodule task state */ 585b0eed085SLongfang Liu switch (vf_pdev->device) { 586b0eed085SLongfang Liu case PCI_DEVICE_ID_HUAWEI_SEC_VF: 587b0eed085SLongfang Liu state = qm_check_reg_state(qm, SEC_CORE_INT_STATUS); 588b0eed085SLongfang Liu if (state) { 589b0eed085SLongfang Liu dev_err(dev, "failed to check QM SEC Core INT state!\n"); 590b0eed085SLongfang Liu return -EBUSY; 591b0eed085SLongfang Liu } 592b0eed085SLongfang Liu return 0; 593b0eed085SLongfang Liu case PCI_DEVICE_ID_HUAWEI_HPRE_VF: 594b0eed085SLongfang Liu state = qm_check_reg_state(qm, HPRE_HAC_INT_STATUS); 595b0eed085SLongfang Liu if (state) { 596b0eed085SLongfang Liu dev_err(dev, "failed to check QM HPRE HAC INT state!\n"); 597b0eed085SLongfang Liu return -EBUSY; 598b0eed085SLongfang Liu } 599b0eed085SLongfang Liu return 0; 600b0eed085SLongfang Liu case PCI_DEVICE_ID_HUAWEI_ZIP_VF: 601b0eed085SLongfang Liu state = qm_check_reg_state(qm, HZIP_CORE_INT_STATUS); 602b0eed085SLongfang Liu if (state) { 603b0eed085SLongfang Liu dev_err(dev, "failed to check QM ZIP Core INT state!\n"); 604b0eed085SLongfang Liu return -EBUSY; 605b0eed085SLongfang Liu } 606b0eed085SLongfang Liu return 0; 607b0eed085SLongfang Liu default: 608b0eed085SLongfang Liu dev_err(dev, "failed to detect acc module type!\n"); 609b0eed085SLongfang Liu return -EINVAL; 610b0eed085SLongfang Liu } 611b0eed085SLongfang Liu } 612b0eed085SLongfang Liu 613b0eed085SLongfang Liu static void hisi_acc_vf_disable_fd(struct hisi_acc_vf_migration_file *migf) 614b0eed085SLongfang Liu { 615b0eed085SLongfang Liu mutex_lock(&migf->lock); 616b0eed085SLongfang Liu migf->disabled = true; 617b0eed085SLongfang Liu migf->total_length = 0; 618b0eed085SLongfang Liu migf->filp->f_pos = 0; 619b0eed085SLongfang Liu mutex_unlock(&migf->lock); 620b0eed085SLongfang Liu } 621b0eed085SLongfang Liu 622b0eed085SLongfang Liu static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vdev) 623b0eed085SLongfang Liu { 624b0eed085SLongfang Liu if (hisi_acc_vdev->resuming_migf) { 625b0eed085SLongfang Liu hisi_acc_vf_disable_fd(hisi_acc_vdev->resuming_migf); 626b0eed085SLongfang Liu fput(hisi_acc_vdev->resuming_migf->filp); 627b0eed085SLongfang Liu hisi_acc_vdev->resuming_migf = NULL; 628b0eed085SLongfang Liu } 629b0eed085SLongfang Liu 630b0eed085SLongfang Liu if (hisi_acc_vdev->saving_migf) { 631b0eed085SLongfang Liu hisi_acc_vf_disable_fd(hisi_acc_vdev->saving_migf); 632b0eed085SLongfang Liu fput(hisi_acc_vdev->saving_migf->filp); 633b0eed085SLongfang Liu hisi_acc_vdev->saving_migf = NULL; 634b0eed085SLongfang Liu } 635b0eed085SLongfang Liu } 636b0eed085SLongfang Liu 6374406f46cSShameer Kolothum /* 6384406f46cSShameer Kolothum * This function is called in all state_mutex unlock cases to 6394406f46cSShameer Kolothum * handle a 'deferred_reset' if exists. 6404406f46cSShameer Kolothum */ 6414406f46cSShameer Kolothum static void 6424406f46cSShameer Kolothum hisi_acc_vf_state_mutex_unlock(struct hisi_acc_vf_core_device *hisi_acc_vdev) 6434406f46cSShameer Kolothum { 6444406f46cSShameer Kolothum again: 6454406f46cSShameer Kolothum spin_lock(&hisi_acc_vdev->reset_lock); 6464406f46cSShameer Kolothum if (hisi_acc_vdev->deferred_reset) { 6474406f46cSShameer Kolothum hisi_acc_vdev->deferred_reset = false; 6484406f46cSShameer Kolothum spin_unlock(&hisi_acc_vdev->reset_lock); 6494406f46cSShameer Kolothum hisi_acc_vdev->vf_qm_state = QM_NOT_READY; 6504406f46cSShameer Kolothum hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; 6514406f46cSShameer Kolothum hisi_acc_vf_disable_fds(hisi_acc_vdev); 6524406f46cSShameer Kolothum goto again; 6534406f46cSShameer Kolothum } 6544406f46cSShameer Kolothum mutex_unlock(&hisi_acc_vdev->state_mutex); 6554406f46cSShameer Kolothum spin_unlock(&hisi_acc_vdev->reset_lock); 6564406f46cSShameer Kolothum } 6574406f46cSShameer Kolothum 658b0eed085SLongfang Liu static void hisi_acc_vf_start_device(struct hisi_acc_vf_core_device *hisi_acc_vdev) 659b0eed085SLongfang Liu { 660b0eed085SLongfang Liu struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; 661b0eed085SLongfang Liu 662b0eed085SLongfang Liu if (hisi_acc_vdev->vf_qm_state != QM_READY) 663b0eed085SLongfang Liu return; 664b0eed085SLongfang Liu 665b0eed085SLongfang Liu vf_qm_fun_reset(hisi_acc_vdev, vf_qm); 666b0eed085SLongfang Liu } 667b0eed085SLongfang Liu 668b0eed085SLongfang Liu static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) 669b0eed085SLongfang Liu { 670b0eed085SLongfang Liu struct device *dev = &hisi_acc_vdev->vf_dev->dev; 671b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev->resuming_migf; 672b0eed085SLongfang Liu int ret; 673b0eed085SLongfang Liu 674b0eed085SLongfang Liu /* Check dev compatibility */ 675b0eed085SLongfang Liu ret = vf_qm_check_match(hisi_acc_vdev, migf); 676b0eed085SLongfang Liu if (ret) { 677b0eed085SLongfang Liu dev_err(dev, "failed to match the VF!\n"); 678b0eed085SLongfang Liu return ret; 679b0eed085SLongfang Liu } 680b0eed085SLongfang Liu /* Recover data to VF */ 681b0eed085SLongfang Liu ret = vf_qm_load_data(hisi_acc_vdev, migf); 682b0eed085SLongfang Liu if (ret) { 683b0eed085SLongfang Liu dev_err(dev, "failed to recover the VF!\n"); 684b0eed085SLongfang Liu return ret; 685b0eed085SLongfang Liu } 686b0eed085SLongfang Liu 687b0eed085SLongfang Liu return 0; 688b0eed085SLongfang Liu } 689b0eed085SLongfang Liu 690b0eed085SLongfang Liu static int hisi_acc_vf_release_file(struct inode *inode, struct file *filp) 691b0eed085SLongfang Liu { 692b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf = filp->private_data; 693b0eed085SLongfang Liu 694b0eed085SLongfang Liu hisi_acc_vf_disable_fd(migf); 695b0eed085SLongfang Liu mutex_destroy(&migf->lock); 696b0eed085SLongfang Liu kfree(migf); 697b0eed085SLongfang Liu return 0; 698b0eed085SLongfang Liu } 699b0eed085SLongfang Liu 700b0eed085SLongfang Liu static ssize_t hisi_acc_vf_resume_write(struct file *filp, const char __user *buf, 701b0eed085SLongfang Liu size_t len, loff_t *pos) 702b0eed085SLongfang Liu { 703b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf = filp->private_data; 704b0eed085SLongfang Liu loff_t requested_length; 705b0eed085SLongfang Liu ssize_t done = 0; 706b0eed085SLongfang Liu int ret; 707b0eed085SLongfang Liu 708b0eed085SLongfang Liu if (pos) 709b0eed085SLongfang Liu return -ESPIPE; 710b0eed085SLongfang Liu pos = &filp->f_pos; 711b0eed085SLongfang Liu 712b0eed085SLongfang Liu if (*pos < 0 || 713b0eed085SLongfang Liu check_add_overflow((loff_t)len, *pos, &requested_length)) 714b0eed085SLongfang Liu return -EINVAL; 715b0eed085SLongfang Liu 716b0eed085SLongfang Liu if (requested_length > sizeof(struct acc_vf_data)) 717b0eed085SLongfang Liu return -ENOMEM; 718b0eed085SLongfang Liu 719b0eed085SLongfang Liu mutex_lock(&migf->lock); 720b0eed085SLongfang Liu if (migf->disabled) { 721b0eed085SLongfang Liu done = -ENODEV; 722b0eed085SLongfang Liu goto out_unlock; 723b0eed085SLongfang Liu } 724b0eed085SLongfang Liu 725b0eed085SLongfang Liu ret = copy_from_user(&migf->vf_data, buf, len); 726b0eed085SLongfang Liu if (ret) { 727b0eed085SLongfang Liu done = -EFAULT; 728b0eed085SLongfang Liu goto out_unlock; 729b0eed085SLongfang Liu } 730b0eed085SLongfang Liu *pos += len; 731b0eed085SLongfang Liu done = len; 732b0eed085SLongfang Liu migf->total_length += len; 733b0eed085SLongfang Liu out_unlock: 734b0eed085SLongfang Liu mutex_unlock(&migf->lock); 735b0eed085SLongfang Liu return done; 736b0eed085SLongfang Liu } 737b0eed085SLongfang Liu 738b0eed085SLongfang Liu static const struct file_operations hisi_acc_vf_resume_fops = { 739b0eed085SLongfang Liu .owner = THIS_MODULE, 740b0eed085SLongfang Liu .write = hisi_acc_vf_resume_write, 741b0eed085SLongfang Liu .release = hisi_acc_vf_release_file, 742b0eed085SLongfang Liu .llseek = no_llseek, 743b0eed085SLongfang Liu }; 744b0eed085SLongfang Liu 745b0eed085SLongfang Liu static struct hisi_acc_vf_migration_file * 746b0eed085SLongfang Liu hisi_acc_vf_pci_resume(struct hisi_acc_vf_core_device *hisi_acc_vdev) 747b0eed085SLongfang Liu { 748b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf; 749b0eed085SLongfang Liu 750b0eed085SLongfang Liu migf = kzalloc(sizeof(*migf), GFP_KERNEL); 751b0eed085SLongfang Liu if (!migf) 752b0eed085SLongfang Liu return ERR_PTR(-ENOMEM); 753b0eed085SLongfang Liu 754b0eed085SLongfang Liu migf->filp = anon_inode_getfile("hisi_acc_vf_mig", &hisi_acc_vf_resume_fops, migf, 755b0eed085SLongfang Liu O_WRONLY); 756b0eed085SLongfang Liu if (IS_ERR(migf->filp)) { 757b0eed085SLongfang Liu int err = PTR_ERR(migf->filp); 758b0eed085SLongfang Liu 759b0eed085SLongfang Liu kfree(migf); 760b0eed085SLongfang Liu return ERR_PTR(err); 761b0eed085SLongfang Liu } 762b0eed085SLongfang Liu 763b0eed085SLongfang Liu stream_open(migf->filp->f_inode, migf->filp); 764b0eed085SLongfang Liu mutex_init(&migf->lock); 765b0eed085SLongfang Liu return migf; 766b0eed085SLongfang Liu } 767b0eed085SLongfang Liu 768b0eed085SLongfang Liu static ssize_t hisi_acc_vf_save_read(struct file *filp, char __user *buf, size_t len, 769b0eed085SLongfang Liu loff_t *pos) 770b0eed085SLongfang Liu { 771b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf = filp->private_data; 772b0eed085SLongfang Liu ssize_t done = 0; 773b0eed085SLongfang Liu int ret; 774b0eed085SLongfang Liu 775b0eed085SLongfang Liu if (pos) 776b0eed085SLongfang Liu return -ESPIPE; 777b0eed085SLongfang Liu pos = &filp->f_pos; 778b0eed085SLongfang Liu 779b0eed085SLongfang Liu mutex_lock(&migf->lock); 780b0eed085SLongfang Liu if (*pos > migf->total_length) { 781b0eed085SLongfang Liu done = -EINVAL; 782b0eed085SLongfang Liu goto out_unlock; 783b0eed085SLongfang Liu } 784b0eed085SLongfang Liu 785b0eed085SLongfang Liu if (migf->disabled) { 786b0eed085SLongfang Liu done = -ENODEV; 787b0eed085SLongfang Liu goto out_unlock; 788b0eed085SLongfang Liu } 789b0eed085SLongfang Liu 790b0eed085SLongfang Liu len = min_t(size_t, migf->total_length - *pos, len); 791b0eed085SLongfang Liu if (len) { 792b0eed085SLongfang Liu ret = copy_to_user(buf, &migf->vf_data, len); 793b0eed085SLongfang Liu if (ret) { 794b0eed085SLongfang Liu done = -EFAULT; 795b0eed085SLongfang Liu goto out_unlock; 796b0eed085SLongfang Liu } 797b0eed085SLongfang Liu *pos += len; 798b0eed085SLongfang Liu done = len; 799b0eed085SLongfang Liu } 800b0eed085SLongfang Liu out_unlock: 801b0eed085SLongfang Liu mutex_unlock(&migf->lock); 802b0eed085SLongfang Liu return done; 803b0eed085SLongfang Liu } 804b0eed085SLongfang Liu 805b0eed085SLongfang Liu static const struct file_operations hisi_acc_vf_save_fops = { 806b0eed085SLongfang Liu .owner = THIS_MODULE, 807b0eed085SLongfang Liu .read = hisi_acc_vf_save_read, 808b0eed085SLongfang Liu .release = hisi_acc_vf_release_file, 809b0eed085SLongfang Liu .llseek = no_llseek, 810b0eed085SLongfang Liu }; 811b0eed085SLongfang Liu 812b0eed085SLongfang Liu static struct hisi_acc_vf_migration_file * 813b0eed085SLongfang Liu hisi_acc_vf_stop_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev) 814b0eed085SLongfang Liu { 815b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf; 816b0eed085SLongfang Liu int ret; 817b0eed085SLongfang Liu 818b0eed085SLongfang Liu migf = kzalloc(sizeof(*migf), GFP_KERNEL); 819b0eed085SLongfang Liu if (!migf) 820b0eed085SLongfang Liu return ERR_PTR(-ENOMEM); 821b0eed085SLongfang Liu 822b0eed085SLongfang Liu migf->filp = anon_inode_getfile("hisi_acc_vf_mig", &hisi_acc_vf_save_fops, migf, 823b0eed085SLongfang Liu O_RDONLY); 824b0eed085SLongfang Liu if (IS_ERR(migf->filp)) { 825b0eed085SLongfang Liu int err = PTR_ERR(migf->filp); 826b0eed085SLongfang Liu 827b0eed085SLongfang Liu kfree(migf); 828b0eed085SLongfang Liu return ERR_PTR(err); 829b0eed085SLongfang Liu } 830b0eed085SLongfang Liu 831b0eed085SLongfang Liu stream_open(migf->filp->f_inode, migf->filp); 832b0eed085SLongfang Liu mutex_init(&migf->lock); 833b0eed085SLongfang Liu 834b0eed085SLongfang Liu ret = vf_qm_state_save(hisi_acc_vdev, migf); 835b0eed085SLongfang Liu if (ret) { 836b0eed085SLongfang Liu fput(migf->filp); 837b0eed085SLongfang Liu return ERR_PTR(ret); 838b0eed085SLongfang Liu } 839b0eed085SLongfang Liu 840b0eed085SLongfang Liu return migf; 841b0eed085SLongfang Liu } 842b0eed085SLongfang Liu 843b0eed085SLongfang Liu static int hisi_acc_vf_stop_device(struct hisi_acc_vf_core_device *hisi_acc_vdev) 844b0eed085SLongfang Liu { 845b0eed085SLongfang Liu struct device *dev = &hisi_acc_vdev->vf_dev->dev; 846b0eed085SLongfang Liu struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; 847b0eed085SLongfang Liu int ret; 848b0eed085SLongfang Liu 849b0eed085SLongfang Liu ret = vf_qm_func_stop(vf_qm); 850b0eed085SLongfang Liu if (ret) { 851b0eed085SLongfang Liu dev_err(dev, "failed to stop QM VF function!\n"); 852b0eed085SLongfang Liu return ret; 853b0eed085SLongfang Liu } 854b0eed085SLongfang Liu 855b0eed085SLongfang Liu ret = hisi_acc_check_int_state(hisi_acc_vdev); 856b0eed085SLongfang Liu if (ret) { 857b0eed085SLongfang Liu dev_err(dev, "failed to check QM INT state!\n"); 858b0eed085SLongfang Liu return ret; 859b0eed085SLongfang Liu } 860b0eed085SLongfang Liu return 0; 861b0eed085SLongfang Liu } 862b0eed085SLongfang Liu 863b0eed085SLongfang Liu static struct file * 864b0eed085SLongfang Liu hisi_acc_vf_set_device_state(struct hisi_acc_vf_core_device *hisi_acc_vdev, 865b0eed085SLongfang Liu u32 new) 866b0eed085SLongfang Liu { 867b0eed085SLongfang Liu u32 cur = hisi_acc_vdev->mig_state; 868b0eed085SLongfang Liu int ret; 869b0eed085SLongfang Liu 870b0eed085SLongfang Liu if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_STOP) { 871b0eed085SLongfang Liu ret = hisi_acc_vf_stop_device(hisi_acc_vdev); 872b0eed085SLongfang Liu if (ret) 873b0eed085SLongfang Liu return ERR_PTR(ret); 874b0eed085SLongfang Liu return NULL; 875b0eed085SLongfang Liu } 876b0eed085SLongfang Liu 877b0eed085SLongfang Liu if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) { 878b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf; 879b0eed085SLongfang Liu 880b0eed085SLongfang Liu migf = hisi_acc_vf_stop_copy(hisi_acc_vdev); 881b0eed085SLongfang Liu if (IS_ERR(migf)) 882b0eed085SLongfang Liu return ERR_CAST(migf); 883b0eed085SLongfang Liu get_file(migf->filp); 884b0eed085SLongfang Liu hisi_acc_vdev->saving_migf = migf; 885b0eed085SLongfang Liu return migf->filp; 886b0eed085SLongfang Liu } 887b0eed085SLongfang Liu 888b0eed085SLongfang Liu if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP)) { 889b0eed085SLongfang Liu hisi_acc_vf_disable_fds(hisi_acc_vdev); 890b0eed085SLongfang Liu return NULL; 891b0eed085SLongfang Liu } 892b0eed085SLongfang Liu 893b0eed085SLongfang Liu if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) { 894b0eed085SLongfang Liu struct hisi_acc_vf_migration_file *migf; 895b0eed085SLongfang Liu 896b0eed085SLongfang Liu migf = hisi_acc_vf_pci_resume(hisi_acc_vdev); 897b0eed085SLongfang Liu if (IS_ERR(migf)) 898b0eed085SLongfang Liu return ERR_CAST(migf); 899b0eed085SLongfang Liu get_file(migf->filp); 900b0eed085SLongfang Liu hisi_acc_vdev->resuming_migf = migf; 901b0eed085SLongfang Liu return migf->filp; 902b0eed085SLongfang Liu } 903b0eed085SLongfang Liu 904b0eed085SLongfang Liu if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) { 905b0eed085SLongfang Liu ret = hisi_acc_vf_load_state(hisi_acc_vdev); 906b0eed085SLongfang Liu if (ret) 907b0eed085SLongfang Liu return ERR_PTR(ret); 908b0eed085SLongfang Liu hisi_acc_vf_disable_fds(hisi_acc_vdev); 909b0eed085SLongfang Liu return NULL; 910b0eed085SLongfang Liu } 911b0eed085SLongfang Liu 912b0eed085SLongfang Liu if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING) { 913b0eed085SLongfang Liu hisi_acc_vf_start_device(hisi_acc_vdev); 914b0eed085SLongfang Liu return NULL; 915b0eed085SLongfang Liu } 916b0eed085SLongfang Liu 917b0eed085SLongfang Liu /* 918b0eed085SLongfang Liu * vfio_mig_get_next_state() does not use arcs other than the above 919b0eed085SLongfang Liu */ 920b0eed085SLongfang Liu WARN_ON(true); 921b0eed085SLongfang Liu return ERR_PTR(-EINVAL); 922b0eed085SLongfang Liu } 923b0eed085SLongfang Liu 924b0eed085SLongfang Liu static struct file * 925b0eed085SLongfang Liu hisi_acc_vfio_pci_set_device_state(struct vfio_device *vdev, 926b0eed085SLongfang Liu enum vfio_device_mig_state new_state) 927b0eed085SLongfang Liu { 928b0eed085SLongfang Liu struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev, 929b0eed085SLongfang Liu struct hisi_acc_vf_core_device, core_device.vdev); 930b0eed085SLongfang Liu enum vfio_device_mig_state next_state; 931b0eed085SLongfang Liu struct file *res = NULL; 932b0eed085SLongfang Liu int ret; 933b0eed085SLongfang Liu 934b0eed085SLongfang Liu mutex_lock(&hisi_acc_vdev->state_mutex); 935b0eed085SLongfang Liu while (new_state != hisi_acc_vdev->mig_state) { 936b0eed085SLongfang Liu ret = vfio_mig_get_next_state(vdev, 937b0eed085SLongfang Liu hisi_acc_vdev->mig_state, 938b0eed085SLongfang Liu new_state, &next_state); 939b0eed085SLongfang Liu if (ret) { 940b0eed085SLongfang Liu res = ERR_PTR(-EINVAL); 941b0eed085SLongfang Liu break; 942b0eed085SLongfang Liu } 943b0eed085SLongfang Liu 944b0eed085SLongfang Liu res = hisi_acc_vf_set_device_state(hisi_acc_vdev, next_state); 945b0eed085SLongfang Liu if (IS_ERR(res)) 946b0eed085SLongfang Liu break; 947b0eed085SLongfang Liu hisi_acc_vdev->mig_state = next_state; 948b0eed085SLongfang Liu if (WARN_ON(res && new_state != hisi_acc_vdev->mig_state)) { 949b0eed085SLongfang Liu fput(res); 950b0eed085SLongfang Liu res = ERR_PTR(-EINVAL); 951b0eed085SLongfang Liu break; 952b0eed085SLongfang Liu } 953b0eed085SLongfang Liu } 9544406f46cSShameer Kolothum hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev); 955b0eed085SLongfang Liu return res; 956b0eed085SLongfang Liu } 957b0eed085SLongfang Liu 958b0eed085SLongfang Liu static int 959b0eed085SLongfang Liu hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev, 960b0eed085SLongfang Liu enum vfio_device_mig_state *curr_state) 961b0eed085SLongfang Liu { 962b0eed085SLongfang Liu struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev, 963b0eed085SLongfang Liu struct hisi_acc_vf_core_device, core_device.vdev); 964b0eed085SLongfang Liu 965b0eed085SLongfang Liu mutex_lock(&hisi_acc_vdev->state_mutex); 966b0eed085SLongfang Liu *curr_state = hisi_acc_vdev->mig_state; 9674406f46cSShameer Kolothum hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev); 968b0eed085SLongfang Liu return 0; 969b0eed085SLongfang Liu } 970b0eed085SLongfang Liu 9714406f46cSShameer Kolothum static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev) 9724406f46cSShameer Kolothum { 973*245898ebSShameer Kolothum struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); 9744406f46cSShameer Kolothum 9754406f46cSShameer Kolothum if (hisi_acc_vdev->core_device.vdev.migration_flags != 9764406f46cSShameer Kolothum VFIO_MIGRATION_STOP_COPY) 9774406f46cSShameer Kolothum return; 9784406f46cSShameer Kolothum 9794406f46cSShameer Kolothum /* 9804406f46cSShameer Kolothum * As the higher VFIO layers are holding locks across reset and using 9814406f46cSShameer Kolothum * those same locks with the mm_lock we need to prevent ABBA deadlock 9824406f46cSShameer Kolothum * with the state_mutex and mm_lock. 9834406f46cSShameer Kolothum * In case the state_mutex was taken already we defer the cleanup work 9844406f46cSShameer Kolothum * to the unlock flow of the other running context. 9854406f46cSShameer Kolothum */ 9864406f46cSShameer Kolothum spin_lock(&hisi_acc_vdev->reset_lock); 9874406f46cSShameer Kolothum hisi_acc_vdev->deferred_reset = true; 9884406f46cSShameer Kolothum if (!mutex_trylock(&hisi_acc_vdev->state_mutex)) { 9894406f46cSShameer Kolothum spin_unlock(&hisi_acc_vdev->reset_lock); 9904406f46cSShameer Kolothum return; 9914406f46cSShameer Kolothum } 9924406f46cSShameer Kolothum spin_unlock(&hisi_acc_vdev->reset_lock); 9934406f46cSShameer Kolothum hisi_acc_vf_state_mutex_unlock(hisi_acc_vdev); 9944406f46cSShameer Kolothum } 9954406f46cSShameer Kolothum 996b0eed085SLongfang Liu static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev) 997b0eed085SLongfang Liu { 998b0eed085SLongfang Liu struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device; 999b0eed085SLongfang Liu struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; 1000b0eed085SLongfang Liu struct pci_dev *vf_dev = vdev->pdev; 1001b0eed085SLongfang Liu 1002b0eed085SLongfang Liu /* 1003b0eed085SLongfang Liu * ACC VF dev BAR2 region consists of both functional register space 1004b0eed085SLongfang Liu * and migration control register space. For migration to work, we 1005b0eed085SLongfang Liu * need access to both. Hence, we map the entire BAR2 region here. 1006b0eed085SLongfang Liu * But unnecessarily exposing the migration BAR region to the Guest 1007b0eed085SLongfang Liu * has the potential to prevent/corrupt the Guest migration. Hence, 1008b0eed085SLongfang Liu * we restrict access to the migration control space from 1009b0eed085SLongfang Liu * Guest(Please see mmap/ioctl/read/write override functions). 1010b0eed085SLongfang Liu * 1011b0eed085SLongfang Liu * Please note that it is OK to expose the entire VF BAR if migration 1012b0eed085SLongfang Liu * is not supported or required as this cannot affect the ACC PF 1013b0eed085SLongfang Liu * configurations. 1014b0eed085SLongfang Liu * 1015b0eed085SLongfang Liu * Also the HiSilicon ACC VF devices supported by this driver on 1016b0eed085SLongfang Liu * HiSilicon hardware platforms are integrated end point devices 1017b0eed085SLongfang Liu * and the platform lacks the capability to perform any PCIe P2P 1018b0eed085SLongfang Liu * between these devices. 1019b0eed085SLongfang Liu */ 1020b0eed085SLongfang Liu 1021b0eed085SLongfang Liu vf_qm->io_base = 1022b0eed085SLongfang Liu ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX), 1023b0eed085SLongfang Liu pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX)); 1024b0eed085SLongfang Liu if (!vf_qm->io_base) 1025b0eed085SLongfang Liu return -EIO; 1026b0eed085SLongfang Liu 1027b0eed085SLongfang Liu vf_qm->fun_type = QM_HW_VF; 1028b0eed085SLongfang Liu vf_qm->pdev = vf_dev; 1029b0eed085SLongfang Liu mutex_init(&vf_qm->mailbox_lock); 1030b0eed085SLongfang Liu 1031b0eed085SLongfang Liu return 0; 1032b0eed085SLongfang Liu } 1033b0eed085SLongfang Liu 1034b0eed085SLongfang Liu static struct hisi_qm *hisi_acc_get_pf_qm(struct pci_dev *pdev) 1035b0eed085SLongfang Liu { 1036b0eed085SLongfang Liu struct hisi_qm *pf_qm; 1037b0eed085SLongfang Liu struct pci_driver *pf_driver; 1038b0eed085SLongfang Liu 1039b0eed085SLongfang Liu if (!pdev->is_virtfn) 1040b0eed085SLongfang Liu return NULL; 1041b0eed085SLongfang Liu 1042b0eed085SLongfang Liu switch (pdev->device) { 1043b0eed085SLongfang Liu case PCI_DEVICE_ID_HUAWEI_SEC_VF: 1044b0eed085SLongfang Liu pf_driver = hisi_sec_get_pf_driver(); 1045b0eed085SLongfang Liu break; 1046b0eed085SLongfang Liu case PCI_DEVICE_ID_HUAWEI_HPRE_VF: 1047b0eed085SLongfang Liu pf_driver = hisi_hpre_get_pf_driver(); 1048b0eed085SLongfang Liu break; 1049b0eed085SLongfang Liu case PCI_DEVICE_ID_HUAWEI_ZIP_VF: 1050b0eed085SLongfang Liu pf_driver = hisi_zip_get_pf_driver(); 1051b0eed085SLongfang Liu break; 1052b0eed085SLongfang Liu default: 1053b0eed085SLongfang Liu return NULL; 1054b0eed085SLongfang Liu } 1055b0eed085SLongfang Liu 1056b0eed085SLongfang Liu if (!pf_driver) 1057b0eed085SLongfang Liu return NULL; 1058b0eed085SLongfang Liu 1059b0eed085SLongfang Liu pf_qm = pci_iov_get_pf_drvdata(pdev, pf_driver); 1060b0eed085SLongfang Liu 1061b0eed085SLongfang Liu return !IS_ERR(pf_qm) ? pf_qm : NULL; 1062b0eed085SLongfang Liu } 1063ee3a5b23SShameer Kolothum 10646abdce51SShameer Kolothum static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev, 10656abdce51SShameer Kolothum size_t count, loff_t *ppos, 10666abdce51SShameer Kolothum size_t *new_count) 10676abdce51SShameer Kolothum { 10686abdce51SShameer Kolothum unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 10696abdce51SShameer Kolothum struct vfio_pci_core_device *vdev = 10706abdce51SShameer Kolothum container_of(core_vdev, struct vfio_pci_core_device, vdev); 10716abdce51SShameer Kolothum 10726abdce51SShameer Kolothum if (index == VFIO_PCI_BAR2_REGION_INDEX) { 10736abdce51SShameer Kolothum loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 10746abdce51SShameer Kolothum resource_size_t end = pci_resource_len(vdev->pdev, index) / 2; 10756abdce51SShameer Kolothum 10766abdce51SShameer Kolothum /* Check if access is for migration control region */ 10776abdce51SShameer Kolothum if (pos >= end) 10786abdce51SShameer Kolothum return -EINVAL; 10796abdce51SShameer Kolothum 10806abdce51SShameer Kolothum *new_count = min(count, (size_t)(end - pos)); 10816abdce51SShameer Kolothum } 10826abdce51SShameer Kolothum 10836abdce51SShameer Kolothum return 0; 10846abdce51SShameer Kolothum } 10856abdce51SShameer Kolothum 10866abdce51SShameer Kolothum static int hisi_acc_vfio_pci_mmap(struct vfio_device *core_vdev, 10876abdce51SShameer Kolothum struct vm_area_struct *vma) 10886abdce51SShameer Kolothum { 10896abdce51SShameer Kolothum struct vfio_pci_core_device *vdev = 10906abdce51SShameer Kolothum container_of(core_vdev, struct vfio_pci_core_device, vdev); 10916abdce51SShameer Kolothum unsigned int index; 10926abdce51SShameer Kolothum 10936abdce51SShameer Kolothum index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 10946abdce51SShameer Kolothum if (index == VFIO_PCI_BAR2_REGION_INDEX) { 10956abdce51SShameer Kolothum u64 req_len, pgoff, req_start; 10966abdce51SShameer Kolothum resource_size_t end = pci_resource_len(vdev->pdev, index) / 2; 10976abdce51SShameer Kolothum 10986abdce51SShameer Kolothum req_len = vma->vm_end - vma->vm_start; 10996abdce51SShameer Kolothum pgoff = vma->vm_pgoff & 11006abdce51SShameer Kolothum ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 11016abdce51SShameer Kolothum req_start = pgoff << PAGE_SHIFT; 11026abdce51SShameer Kolothum 11036abdce51SShameer Kolothum if (req_start + req_len > end) 11046abdce51SShameer Kolothum return -EINVAL; 11056abdce51SShameer Kolothum } 11066abdce51SShameer Kolothum 11076abdce51SShameer Kolothum return vfio_pci_core_mmap(core_vdev, vma); 11086abdce51SShameer Kolothum } 11096abdce51SShameer Kolothum 11106abdce51SShameer Kolothum static ssize_t hisi_acc_vfio_pci_write(struct vfio_device *core_vdev, 11116abdce51SShameer Kolothum const char __user *buf, size_t count, 11126abdce51SShameer Kolothum loff_t *ppos) 11136abdce51SShameer Kolothum { 11146abdce51SShameer Kolothum size_t new_count = count; 11156abdce51SShameer Kolothum int ret; 11166abdce51SShameer Kolothum 11176abdce51SShameer Kolothum ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, &new_count); 11186abdce51SShameer Kolothum if (ret) 11196abdce51SShameer Kolothum return ret; 11206abdce51SShameer Kolothum 11216abdce51SShameer Kolothum return vfio_pci_core_write(core_vdev, buf, new_count, ppos); 11226abdce51SShameer Kolothum } 11236abdce51SShameer Kolothum 11246abdce51SShameer Kolothum static ssize_t hisi_acc_vfio_pci_read(struct vfio_device *core_vdev, 11256abdce51SShameer Kolothum char __user *buf, size_t count, 11266abdce51SShameer Kolothum loff_t *ppos) 11276abdce51SShameer Kolothum { 11286abdce51SShameer Kolothum size_t new_count = count; 11296abdce51SShameer Kolothum int ret; 11306abdce51SShameer Kolothum 11316abdce51SShameer Kolothum ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, &new_count); 11326abdce51SShameer Kolothum if (ret) 11336abdce51SShameer Kolothum return ret; 11346abdce51SShameer Kolothum 11356abdce51SShameer Kolothum return vfio_pci_core_read(core_vdev, buf, new_count, ppos); 11366abdce51SShameer Kolothum } 11376abdce51SShameer Kolothum 11386abdce51SShameer Kolothum static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int cmd, 11396abdce51SShameer Kolothum unsigned long arg) 11406abdce51SShameer Kolothum { 11416abdce51SShameer Kolothum if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 11426abdce51SShameer Kolothum struct vfio_pci_core_device *vdev = 11436abdce51SShameer Kolothum container_of(core_vdev, struct vfio_pci_core_device, vdev); 11446abdce51SShameer Kolothum struct pci_dev *pdev = vdev->pdev; 11456abdce51SShameer Kolothum struct vfio_region_info info; 11466abdce51SShameer Kolothum unsigned long minsz; 11476abdce51SShameer Kolothum 11486abdce51SShameer Kolothum minsz = offsetofend(struct vfio_region_info, offset); 11496abdce51SShameer Kolothum 11506abdce51SShameer Kolothum if (copy_from_user(&info, (void __user *)arg, minsz)) 11516abdce51SShameer Kolothum return -EFAULT; 11526abdce51SShameer Kolothum 11536abdce51SShameer Kolothum if (info.argsz < minsz) 11546abdce51SShameer Kolothum return -EINVAL; 11556abdce51SShameer Kolothum 11566abdce51SShameer Kolothum if (info.index == VFIO_PCI_BAR2_REGION_INDEX) { 11576abdce51SShameer Kolothum info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 11586abdce51SShameer Kolothum 11596abdce51SShameer Kolothum /* 11606abdce51SShameer Kolothum * ACC VF dev BAR2 region consists of both functional 11616abdce51SShameer Kolothum * register space and migration control register space. 11626abdce51SShameer Kolothum * Report only the functional region to Guest. 11636abdce51SShameer Kolothum */ 11646abdce51SShameer Kolothum info.size = pci_resource_len(pdev, info.index) / 2; 11656abdce51SShameer Kolothum 11666abdce51SShameer Kolothum info.flags = VFIO_REGION_INFO_FLAG_READ | 11676abdce51SShameer Kolothum VFIO_REGION_INFO_FLAG_WRITE | 11686abdce51SShameer Kolothum VFIO_REGION_INFO_FLAG_MMAP; 11696abdce51SShameer Kolothum 11706abdce51SShameer Kolothum return copy_to_user((void __user *)arg, &info, minsz) ? 11716abdce51SShameer Kolothum -EFAULT : 0; 11726abdce51SShameer Kolothum } 11736abdce51SShameer Kolothum } 11746abdce51SShameer Kolothum return vfio_pci_core_ioctl(core_vdev, cmd, arg); 11756abdce51SShameer Kolothum } 11766abdce51SShameer Kolothum 1177ee3a5b23SShameer Kolothum static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev) 1178ee3a5b23SShameer Kolothum { 1179b0eed085SLongfang Liu struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev, 1180b0eed085SLongfang Liu struct hisi_acc_vf_core_device, core_device.vdev); 1181b0eed085SLongfang Liu struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device; 1182ee3a5b23SShameer Kolothum int ret; 1183ee3a5b23SShameer Kolothum 1184ee3a5b23SShameer Kolothum ret = vfio_pci_core_enable(vdev); 1185ee3a5b23SShameer Kolothum if (ret) 1186ee3a5b23SShameer Kolothum return ret; 1187ee3a5b23SShameer Kolothum 11886e97eba8SYishai Hadas if (core_vdev->mig_ops) { 1189b0eed085SLongfang Liu ret = hisi_acc_vf_qm_init(hisi_acc_vdev); 1190b0eed085SLongfang Liu if (ret) { 1191b0eed085SLongfang Liu vfio_pci_core_disable(vdev); 1192b0eed085SLongfang Liu return ret; 1193b0eed085SLongfang Liu } 1194b0eed085SLongfang Liu hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; 1195b0eed085SLongfang Liu } 1196ee3a5b23SShameer Kolothum 1197b0eed085SLongfang Liu vfio_pci_core_finish_enable(vdev); 1198ee3a5b23SShameer Kolothum return 0; 1199ee3a5b23SShameer Kolothum } 1200ee3a5b23SShameer Kolothum 1201b0eed085SLongfang Liu static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev) 1202b0eed085SLongfang Liu { 1203b0eed085SLongfang Liu struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev, 1204b0eed085SLongfang Liu struct hisi_acc_vf_core_device, core_device.vdev); 1205b0eed085SLongfang Liu struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; 1206b0eed085SLongfang Liu 1207b0eed085SLongfang Liu iounmap(vf_qm->io_base); 1208b0eed085SLongfang Liu vfio_pci_core_close_device(core_vdev); 1209b0eed085SLongfang Liu } 1210b0eed085SLongfang Liu 12116e97eba8SYishai Hadas static const struct vfio_migration_ops hisi_acc_vfio_pci_migrn_state_ops = { 12126e97eba8SYishai Hadas .migration_set_state = hisi_acc_vfio_pci_set_device_state, 12136e97eba8SYishai Hadas .migration_get_state = hisi_acc_vfio_pci_get_device_state, 12146e97eba8SYishai Hadas }; 12156e97eba8SYishai Hadas 12166abdce51SShameer Kolothum static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = { 12176abdce51SShameer Kolothum .name = "hisi-acc-vfio-pci-migration", 12186abdce51SShameer Kolothum .open_device = hisi_acc_vfio_pci_open_device, 1219b0eed085SLongfang Liu .close_device = hisi_acc_vfio_pci_close_device, 12206abdce51SShameer Kolothum .ioctl = hisi_acc_vfio_pci_ioctl, 12216abdce51SShameer Kolothum .device_feature = vfio_pci_core_ioctl_feature, 12226abdce51SShameer Kolothum .read = hisi_acc_vfio_pci_read, 12236abdce51SShameer Kolothum .write = hisi_acc_vfio_pci_write, 12246abdce51SShameer Kolothum .mmap = hisi_acc_vfio_pci_mmap, 12256abdce51SShameer Kolothum .request = vfio_pci_core_request, 12266abdce51SShameer Kolothum .match = vfio_pci_core_match, 12276abdce51SShameer Kolothum }; 12286abdce51SShameer Kolothum 1229ee3a5b23SShameer Kolothum static const struct vfio_device_ops hisi_acc_vfio_pci_ops = { 1230ee3a5b23SShameer Kolothum .name = "hisi-acc-vfio-pci", 1231ee3a5b23SShameer Kolothum .open_device = hisi_acc_vfio_pci_open_device, 1232ee3a5b23SShameer Kolothum .close_device = vfio_pci_core_close_device, 1233ee3a5b23SShameer Kolothum .ioctl = vfio_pci_core_ioctl, 1234ee3a5b23SShameer Kolothum .device_feature = vfio_pci_core_ioctl_feature, 1235ee3a5b23SShameer Kolothum .read = vfio_pci_core_read, 1236ee3a5b23SShameer Kolothum .write = vfio_pci_core_write, 1237ee3a5b23SShameer Kolothum .mmap = vfio_pci_core_mmap, 1238ee3a5b23SShameer Kolothum .request = vfio_pci_core_request, 1239ee3a5b23SShameer Kolothum .match = vfio_pci_core_match, 1240ee3a5b23SShameer Kolothum }; 1241ee3a5b23SShameer Kolothum 1242b0eed085SLongfang Liu static int 1243b0eed085SLongfang Liu hisi_acc_vfio_pci_migrn_init(struct hisi_acc_vf_core_device *hisi_acc_vdev, 1244b0eed085SLongfang Liu struct pci_dev *pdev, struct hisi_qm *pf_qm) 1245b0eed085SLongfang Liu { 1246b0eed085SLongfang Liu int vf_id; 1247b0eed085SLongfang Liu 1248b0eed085SLongfang Liu vf_id = pci_iov_vf_id(pdev); 1249b0eed085SLongfang Liu if (vf_id < 0) 1250b0eed085SLongfang Liu return vf_id; 1251b0eed085SLongfang Liu 1252b0eed085SLongfang Liu hisi_acc_vdev->vf_id = vf_id + 1; 1253b0eed085SLongfang Liu hisi_acc_vdev->core_device.vdev.migration_flags = 1254b0eed085SLongfang Liu VFIO_MIGRATION_STOP_COPY; 1255b0eed085SLongfang Liu hisi_acc_vdev->pf_qm = pf_qm; 1256b0eed085SLongfang Liu hisi_acc_vdev->vf_dev = pdev; 1257b0eed085SLongfang Liu mutex_init(&hisi_acc_vdev->state_mutex); 1258b0eed085SLongfang Liu 1259b0eed085SLongfang Liu return 0; 1260b0eed085SLongfang Liu } 1261b0eed085SLongfang Liu 1262ee3a5b23SShameer Kolothum static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1263ee3a5b23SShameer Kolothum { 1264b0eed085SLongfang Liu struct hisi_acc_vf_core_device *hisi_acc_vdev; 1265b0eed085SLongfang Liu struct hisi_qm *pf_qm; 1266ee3a5b23SShameer Kolothum int ret; 1267ee3a5b23SShameer Kolothum 1268b0eed085SLongfang Liu hisi_acc_vdev = kzalloc(sizeof(*hisi_acc_vdev), GFP_KERNEL); 1269b0eed085SLongfang Liu if (!hisi_acc_vdev) 1270ee3a5b23SShameer Kolothum return -ENOMEM; 1271ee3a5b23SShameer Kolothum 1272b0eed085SLongfang Liu pf_qm = hisi_acc_get_pf_qm(pdev); 1273b0eed085SLongfang Liu if (pf_qm && pf_qm->ver >= QM_HW_V3) { 1274b0eed085SLongfang Liu ret = hisi_acc_vfio_pci_migrn_init(hisi_acc_vdev, pdev, pf_qm); 1275b0eed085SLongfang Liu if (!ret) { 1276b0eed085SLongfang Liu vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev, 1277b0eed085SLongfang Liu &hisi_acc_vfio_pci_migrn_ops); 12786e97eba8SYishai Hadas hisi_acc_vdev->core_device.vdev.mig_ops = 12796e97eba8SYishai Hadas &hisi_acc_vfio_pci_migrn_state_ops; 1280b0eed085SLongfang Liu } else { 1281b0eed085SLongfang Liu pci_warn(pdev, "migration support failed, continue with generic interface\n"); 1282b0eed085SLongfang Liu vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev, 1283b0eed085SLongfang Liu &hisi_acc_vfio_pci_ops); 1284b0eed085SLongfang Liu } 1285b0eed085SLongfang Liu } else { 1286b0eed085SLongfang Liu vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev, 1287b0eed085SLongfang Liu &hisi_acc_vfio_pci_ops); 1288b0eed085SLongfang Liu } 1289ee3a5b23SShameer Kolothum 129091be0bd6SJason Gunthorpe dev_set_drvdata(&pdev->dev, &hisi_acc_vdev->core_device); 1291b0eed085SLongfang Liu ret = vfio_pci_core_register_device(&hisi_acc_vdev->core_device); 1292ee3a5b23SShameer Kolothum if (ret) 1293ee3a5b23SShameer Kolothum goto out_free; 1294ee3a5b23SShameer Kolothum return 0; 1295ee3a5b23SShameer Kolothum 1296ee3a5b23SShameer Kolothum out_free: 1297b0eed085SLongfang Liu vfio_pci_core_uninit_device(&hisi_acc_vdev->core_device); 1298b0eed085SLongfang Liu kfree(hisi_acc_vdev); 1299ee3a5b23SShameer Kolothum return ret; 1300ee3a5b23SShameer Kolothum } 1301ee3a5b23SShameer Kolothum 1302ee3a5b23SShameer Kolothum static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev) 1303ee3a5b23SShameer Kolothum { 1304*245898ebSShameer Kolothum struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); 1305ee3a5b23SShameer Kolothum 1306b0eed085SLongfang Liu vfio_pci_core_unregister_device(&hisi_acc_vdev->core_device); 1307b0eed085SLongfang Liu vfio_pci_core_uninit_device(&hisi_acc_vdev->core_device); 1308b0eed085SLongfang Liu kfree(hisi_acc_vdev); 1309ee3a5b23SShameer Kolothum } 1310ee3a5b23SShameer Kolothum 1311ee3a5b23SShameer Kolothum static const struct pci_device_id hisi_acc_vfio_pci_table[] = { 1312ee3a5b23SShameer Kolothum { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) }, 1313ee3a5b23SShameer Kolothum { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_VF) }, 1314ee3a5b23SShameer Kolothum { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) }, 1315ee3a5b23SShameer Kolothum { } 1316ee3a5b23SShameer Kolothum }; 1317ee3a5b23SShameer Kolothum 1318ee3a5b23SShameer Kolothum MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table); 1319ee3a5b23SShameer Kolothum 13204406f46cSShameer Kolothum static const struct pci_error_handlers hisi_acc_vf_err_handlers = { 13214406f46cSShameer Kolothum .reset_done = hisi_acc_vf_pci_aer_reset_done, 13224406f46cSShameer Kolothum .error_detected = vfio_pci_core_aer_err_detected, 13234406f46cSShameer Kolothum }; 13244406f46cSShameer Kolothum 1325ee3a5b23SShameer Kolothum static struct pci_driver hisi_acc_vfio_pci_driver = { 1326ee3a5b23SShameer Kolothum .name = KBUILD_MODNAME, 1327ee3a5b23SShameer Kolothum .id_table = hisi_acc_vfio_pci_table, 1328ee3a5b23SShameer Kolothum .probe = hisi_acc_vfio_pci_probe, 1329ee3a5b23SShameer Kolothum .remove = hisi_acc_vfio_pci_remove, 13304406f46cSShameer Kolothum .err_handler = &hisi_acc_vf_err_handlers, 1331c490513cSJason Gunthorpe .driver_managed_dma = true, 1332ee3a5b23SShameer Kolothum }; 1333ee3a5b23SShameer Kolothum 1334ee3a5b23SShameer Kolothum module_pci_driver(hisi_acc_vfio_pci_driver); 1335ee3a5b23SShameer Kolothum 1336ee3a5b23SShameer Kolothum MODULE_LICENSE("GPL v2"); 1337ee3a5b23SShameer Kolothum MODULE_AUTHOR("Liu Longfang <liulongfang@huawei.com>"); 1338ee3a5b23SShameer Kolothum MODULE_AUTHOR("Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>"); 1339b0eed085SLongfang Liu MODULE_DESCRIPTION("HiSilicon VFIO PCI - VFIO PCI driver with live migration support for HiSilicon ACC device family"); 1340