164b9f64fSJason Wang // SPDX-License-Identifier: GPL-2.0-only 264b9f64fSJason Wang /* 364b9f64fSJason Wang * vDPA bridge driver for modern virtio-pci device 464b9f64fSJason Wang * 564b9f64fSJason Wang * Copyright (c) 2020, Red Hat Inc. All rights reserved. 664b9f64fSJason Wang * Author: Jason Wang <jasowang@redhat.com> 764b9f64fSJason Wang * 864b9f64fSJason Wang * Based on virtio_pci_modern.c. 964b9f64fSJason Wang */ 1064b9f64fSJason Wang 1164b9f64fSJason Wang #include <linux/interrupt.h> 1264b9f64fSJason Wang #include <linux/module.h> 1364b9f64fSJason Wang #include <linux/pci.h> 1464b9f64fSJason Wang #include <linux/vdpa.h> 1564b9f64fSJason Wang #include <linux/virtio.h> 1664b9f64fSJason Wang #include <linux/virtio_config.h> 1764b9f64fSJason Wang #include <linux/virtio_ring.h> 1864b9f64fSJason Wang #include <linux/virtio_pci.h> 1964b9f64fSJason Wang #include <linux/virtio_pci_modern.h> 2064b9f64fSJason Wang 2164b9f64fSJason Wang #define VP_VDPA_QUEUE_MAX 256 2264b9f64fSJason Wang #define VP_VDPA_DRIVER_NAME "vp_vdpa" 2364b9f64fSJason Wang #define VP_VDPA_NAME_SIZE 256 2464b9f64fSJason Wang 2564b9f64fSJason Wang struct vp_vring { 2664b9f64fSJason Wang void __iomem *notify; 2764b9f64fSJason Wang char msix_name[VP_VDPA_NAME_SIZE]; 2864b9f64fSJason Wang struct vdpa_callback cb; 29526cb858SJason Wang resource_size_t notify_pa; 3064b9f64fSJason Wang int irq; 3164b9f64fSJason Wang }; 3264b9f64fSJason Wang 3364b9f64fSJason Wang struct vp_vdpa { 3464b9f64fSJason Wang struct vdpa_device vdpa; 3564b9f64fSJason Wang struct virtio_pci_modern_device mdev; 3664b9f64fSJason Wang struct vp_vring *vring; 3764b9f64fSJason Wang struct vdpa_callback config_cb; 3864b9f64fSJason Wang char msix_name[VP_VDPA_NAME_SIZE]; 3964b9f64fSJason Wang int config_irq; 4064b9f64fSJason Wang int queues; 4164b9f64fSJason Wang int vectors; 4264b9f64fSJason Wang }; 4364b9f64fSJason Wang 4464b9f64fSJason Wang static struct vp_vdpa *vdpa_to_vp(struct vdpa_device *vdpa) 4564b9f64fSJason Wang { 4664b9f64fSJason Wang return container_of(vdpa, struct vp_vdpa, vdpa); 4764b9f64fSJason Wang } 4864b9f64fSJason Wang 4964b9f64fSJason Wang static struct virtio_pci_modern_device *vdpa_to_mdev(struct vdpa_device *vdpa) 5064b9f64fSJason Wang { 5164b9f64fSJason Wang struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 5264b9f64fSJason Wang 5364b9f64fSJason Wang return &vp_vdpa->mdev; 5464b9f64fSJason Wang } 5564b9f64fSJason Wang 5664b9f64fSJason Wang static u64 vp_vdpa_get_features(struct vdpa_device *vdpa) 5764b9f64fSJason Wang { 5864b9f64fSJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 5964b9f64fSJason Wang 6064b9f64fSJason Wang return vp_modern_get_features(mdev); 6164b9f64fSJason Wang } 6264b9f64fSJason Wang 6364b9f64fSJason Wang static int vp_vdpa_set_features(struct vdpa_device *vdpa, u64 features) 6464b9f64fSJason Wang { 6564b9f64fSJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 6664b9f64fSJason Wang 6764b9f64fSJason Wang vp_modern_set_features(mdev, features); 6864b9f64fSJason Wang 6964b9f64fSJason Wang return 0; 7064b9f64fSJason Wang } 7164b9f64fSJason Wang 7264b9f64fSJason Wang static u8 vp_vdpa_get_status(struct vdpa_device *vdpa) 7364b9f64fSJason Wang { 7464b9f64fSJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 7564b9f64fSJason Wang 7664b9f64fSJason Wang return vp_modern_get_status(mdev); 7764b9f64fSJason Wang } 7864b9f64fSJason Wang 7964b9f64fSJason Wang static void vp_vdpa_free_irq(struct vp_vdpa *vp_vdpa) 8064b9f64fSJason Wang { 8164b9f64fSJason Wang struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; 8264b9f64fSJason Wang struct pci_dev *pdev = mdev->pci_dev; 8364b9f64fSJason Wang int i; 8464b9f64fSJason Wang 8564b9f64fSJason Wang for (i = 0; i < vp_vdpa->queues; i++) { 8664b9f64fSJason Wang if (vp_vdpa->vring[i].irq != VIRTIO_MSI_NO_VECTOR) { 8764b9f64fSJason Wang vp_modern_queue_vector(mdev, i, VIRTIO_MSI_NO_VECTOR); 8864b9f64fSJason Wang devm_free_irq(&pdev->dev, vp_vdpa->vring[i].irq, 8964b9f64fSJason Wang &vp_vdpa->vring[i]); 9064b9f64fSJason Wang vp_vdpa->vring[i].irq = VIRTIO_MSI_NO_VECTOR; 9164b9f64fSJason Wang } 9264b9f64fSJason Wang } 9364b9f64fSJason Wang 9464b9f64fSJason Wang if (vp_vdpa->config_irq != VIRTIO_MSI_NO_VECTOR) { 9564b9f64fSJason Wang vp_modern_config_vector(mdev, VIRTIO_MSI_NO_VECTOR); 9664b9f64fSJason Wang devm_free_irq(&pdev->dev, vp_vdpa->config_irq, vp_vdpa); 9764b9f64fSJason Wang vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR; 9864b9f64fSJason Wang } 9964b9f64fSJason Wang 10064b9f64fSJason Wang if (vp_vdpa->vectors) { 10164b9f64fSJason Wang pci_free_irq_vectors(pdev); 10264b9f64fSJason Wang vp_vdpa->vectors = 0; 10364b9f64fSJason Wang } 10464b9f64fSJason Wang } 10564b9f64fSJason Wang 10664b9f64fSJason Wang static irqreturn_t vp_vdpa_vq_handler(int irq, void *arg) 10764b9f64fSJason Wang { 10864b9f64fSJason Wang struct vp_vring *vring = arg; 10964b9f64fSJason Wang 11064b9f64fSJason Wang if (vring->cb.callback) 11164b9f64fSJason Wang return vring->cb.callback(vring->cb.private); 11264b9f64fSJason Wang 11364b9f64fSJason Wang return IRQ_HANDLED; 11464b9f64fSJason Wang } 11564b9f64fSJason Wang 11664b9f64fSJason Wang static irqreturn_t vp_vdpa_config_handler(int irq, void *arg) 11764b9f64fSJason Wang { 11864b9f64fSJason Wang struct vp_vdpa *vp_vdpa = arg; 11964b9f64fSJason Wang 12064b9f64fSJason Wang if (vp_vdpa->config_cb.callback) 12164b9f64fSJason Wang return vp_vdpa->config_cb.callback(vp_vdpa->config_cb.private); 12264b9f64fSJason Wang 12364b9f64fSJason Wang return IRQ_HANDLED; 12464b9f64fSJason Wang } 12564b9f64fSJason Wang 12664b9f64fSJason Wang static int vp_vdpa_request_irq(struct vp_vdpa *vp_vdpa) 12764b9f64fSJason Wang { 12864b9f64fSJason Wang struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; 12964b9f64fSJason Wang struct pci_dev *pdev = mdev->pci_dev; 13064b9f64fSJason Wang int i, ret, irq; 13164b9f64fSJason Wang int queues = vp_vdpa->queues; 13264b9f64fSJason Wang int vectors = queues + 1; 13364b9f64fSJason Wang 13464b9f64fSJason Wang ret = pci_alloc_irq_vectors(pdev, vectors, vectors, PCI_IRQ_MSIX); 13564b9f64fSJason Wang if (ret != vectors) { 13664b9f64fSJason Wang dev_err(&pdev->dev, 13764b9f64fSJason Wang "vp_vdpa: fail to allocate irq vectors want %d but %d\n", 13864b9f64fSJason Wang vectors, ret); 13964b9f64fSJason Wang return ret; 14064b9f64fSJason Wang } 14164b9f64fSJason Wang 14264b9f64fSJason Wang vp_vdpa->vectors = vectors; 14364b9f64fSJason Wang 14464b9f64fSJason Wang for (i = 0; i < queues; i++) { 14564b9f64fSJason Wang snprintf(vp_vdpa->vring[i].msix_name, VP_VDPA_NAME_SIZE, 14664b9f64fSJason Wang "vp-vdpa[%s]-%d\n", pci_name(pdev), i); 14764b9f64fSJason Wang irq = pci_irq_vector(pdev, i); 14864b9f64fSJason Wang ret = devm_request_irq(&pdev->dev, irq, 14964b9f64fSJason Wang vp_vdpa_vq_handler, 15064b9f64fSJason Wang 0, vp_vdpa->vring[i].msix_name, 15164b9f64fSJason Wang &vp_vdpa->vring[i]); 15264b9f64fSJason Wang if (ret) { 15364b9f64fSJason Wang dev_err(&pdev->dev, 15464b9f64fSJason Wang "vp_vdpa: fail to request irq for vq %d\n", i); 15564b9f64fSJason Wang goto err; 15664b9f64fSJason Wang } 15764b9f64fSJason Wang vp_modern_queue_vector(mdev, i, i); 15864b9f64fSJason Wang vp_vdpa->vring[i].irq = irq; 15964b9f64fSJason Wang } 16064b9f64fSJason Wang 16164b9f64fSJason Wang snprintf(vp_vdpa->msix_name, VP_VDPA_NAME_SIZE, "vp-vdpa[%s]-config\n", 16264b9f64fSJason Wang pci_name(pdev)); 16364b9f64fSJason Wang irq = pci_irq_vector(pdev, queues); 16464b9f64fSJason Wang ret = devm_request_irq(&pdev->dev, irq, vp_vdpa_config_handler, 0, 16564b9f64fSJason Wang vp_vdpa->msix_name, vp_vdpa); 16664b9f64fSJason Wang if (ret) { 16764b9f64fSJason Wang dev_err(&pdev->dev, 16864b9f64fSJason Wang "vp_vdpa: fail to request irq for vq %d\n", i); 16964b9f64fSJason Wang goto err; 17064b9f64fSJason Wang } 17164b9f64fSJason Wang vp_modern_config_vector(mdev, queues); 17264b9f64fSJason Wang vp_vdpa->config_irq = irq; 17364b9f64fSJason Wang 17464b9f64fSJason Wang return 0; 17564b9f64fSJason Wang err: 17664b9f64fSJason Wang vp_vdpa_free_irq(vp_vdpa); 17764b9f64fSJason Wang return ret; 17864b9f64fSJason Wang } 17964b9f64fSJason Wang 18064b9f64fSJason Wang static void vp_vdpa_set_status(struct vdpa_device *vdpa, u8 status) 18164b9f64fSJason Wang { 18264b9f64fSJason Wang struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 18364b9f64fSJason Wang struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; 18464b9f64fSJason Wang u8 s = vp_vdpa_get_status(vdpa); 18564b9f64fSJason Wang 18664b9f64fSJason Wang if (status & VIRTIO_CONFIG_S_DRIVER_OK && 18764b9f64fSJason Wang !(s & VIRTIO_CONFIG_S_DRIVER_OK)) { 18864b9f64fSJason Wang vp_vdpa_request_irq(vp_vdpa); 18964b9f64fSJason Wang } 19064b9f64fSJason Wang 19164b9f64fSJason Wang vp_modern_set_status(mdev, status); 19264b9f64fSJason Wang 19364b9f64fSJason Wang if (!(status & VIRTIO_CONFIG_S_DRIVER_OK) && 19464b9f64fSJason Wang (s & VIRTIO_CONFIG_S_DRIVER_OK)) 19564b9f64fSJason Wang vp_vdpa_free_irq(vp_vdpa); 19664b9f64fSJason Wang } 19764b9f64fSJason Wang 19864b9f64fSJason Wang static u16 vp_vdpa_get_vq_num_max(struct vdpa_device *vdpa) 19964b9f64fSJason Wang { 20064b9f64fSJason Wang return VP_VDPA_QUEUE_MAX; 20164b9f64fSJason Wang } 20264b9f64fSJason Wang 20364b9f64fSJason Wang static int vp_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 qid, 20464b9f64fSJason Wang struct vdpa_vq_state *state) 20564b9f64fSJason Wang { 20664b9f64fSJason Wang /* Note that this is not supported by virtio specification, so 20764b9f64fSJason Wang * we return -EOPNOTSUPP here. This means we can't support live 20864b9f64fSJason Wang * migration, vhost device start/stop. 20964b9f64fSJason Wang */ 21064b9f64fSJason Wang return -EOPNOTSUPP; 21164b9f64fSJason Wang } 21264b9f64fSJason Wang 213*1225c216SJason Wang static int vp_vdpa_set_vq_state_split(struct vdpa_device *vdpa, 214*1225c216SJason Wang const struct vdpa_vq_state *state) 215*1225c216SJason Wang { 216*1225c216SJason Wang const struct vdpa_vq_state_split *split = &state->split; 217*1225c216SJason Wang 218*1225c216SJason Wang if (split->avail_index == 0) 219*1225c216SJason Wang return 0; 220*1225c216SJason Wang 221*1225c216SJason Wang return -EOPNOTSUPP; 222*1225c216SJason Wang } 223*1225c216SJason Wang 224*1225c216SJason Wang static int vp_vdpa_set_vq_state_packed(struct vdpa_device *vdpa, 225*1225c216SJason Wang const struct vdpa_vq_state *state) 226*1225c216SJason Wang { 227*1225c216SJason Wang const struct vdpa_vq_state_packed *packed = &state->packed; 228*1225c216SJason Wang 229*1225c216SJason Wang if (packed->last_avail_counter == 1 && 230*1225c216SJason Wang packed->last_avail_idx == 0 && 231*1225c216SJason Wang packed->last_used_counter == 1 && 232*1225c216SJason Wang packed->last_used_idx == 0) 233*1225c216SJason Wang return 0; 234*1225c216SJason Wang 235*1225c216SJason Wang return -EOPNOTSUPP; 236*1225c216SJason Wang } 237*1225c216SJason Wang 23864b9f64fSJason Wang static int vp_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 qid, 23964b9f64fSJason Wang const struct vdpa_vq_state *state) 24064b9f64fSJason Wang { 241*1225c216SJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 242*1225c216SJason Wang 243*1225c216SJason Wang /* Note that this is not supported by virtio specification. 244*1225c216SJason Wang * But if the state is by chance equal to the device initial 245*1225c216SJason Wang * state, we can let it go. 24664b9f64fSJason Wang */ 247*1225c216SJason Wang if ((vp_modern_get_status(mdev) & VIRTIO_CONFIG_S_FEATURES_OK) && 248*1225c216SJason Wang !vp_modern_get_queue_enable(mdev, qid)) { 249*1225c216SJason Wang if (vp_modern_get_driver_features(mdev) & 250*1225c216SJason Wang BIT_ULL(VIRTIO_F_RING_PACKED)) 251*1225c216SJason Wang return vp_vdpa_set_vq_state_packed(vdpa, state); 252*1225c216SJason Wang else 253*1225c216SJason Wang return vp_vdpa_set_vq_state_split(vdpa, state); 254*1225c216SJason Wang } 255*1225c216SJason Wang 25664b9f64fSJason Wang return -EOPNOTSUPP; 25764b9f64fSJason Wang } 25864b9f64fSJason Wang 25964b9f64fSJason Wang static void vp_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 qid, 26064b9f64fSJason Wang struct vdpa_callback *cb) 26164b9f64fSJason Wang { 26264b9f64fSJason Wang struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 26364b9f64fSJason Wang 26464b9f64fSJason Wang vp_vdpa->vring[qid].cb = *cb; 26564b9f64fSJason Wang } 26664b9f64fSJason Wang 26764b9f64fSJason Wang static void vp_vdpa_set_vq_ready(struct vdpa_device *vdpa, 26864b9f64fSJason Wang u16 qid, bool ready) 26964b9f64fSJason Wang { 27064b9f64fSJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 27164b9f64fSJason Wang 27264b9f64fSJason Wang vp_modern_set_queue_enable(mdev, qid, ready); 27364b9f64fSJason Wang } 27464b9f64fSJason Wang 27564b9f64fSJason Wang static bool vp_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 qid) 27664b9f64fSJason Wang { 27764b9f64fSJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 27864b9f64fSJason Wang 27964b9f64fSJason Wang return vp_modern_get_queue_enable(mdev, qid); 28064b9f64fSJason Wang } 28164b9f64fSJason Wang 28264b9f64fSJason Wang static void vp_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 qid, 28364b9f64fSJason Wang u32 num) 28464b9f64fSJason Wang { 28564b9f64fSJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 28664b9f64fSJason Wang 28764b9f64fSJason Wang vp_modern_set_queue_size(mdev, qid, num); 28864b9f64fSJason Wang } 28964b9f64fSJason Wang 29064b9f64fSJason Wang static int vp_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 qid, 29164b9f64fSJason Wang u64 desc_area, u64 driver_area, 29264b9f64fSJason Wang u64 device_area) 29364b9f64fSJason Wang { 29464b9f64fSJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 29564b9f64fSJason Wang 29664b9f64fSJason Wang vp_modern_queue_address(mdev, qid, desc_area, 29764b9f64fSJason Wang driver_area, device_area); 29864b9f64fSJason Wang 29964b9f64fSJason Wang return 0; 30064b9f64fSJason Wang } 30164b9f64fSJason Wang 30264b9f64fSJason Wang static void vp_vdpa_kick_vq(struct vdpa_device *vdpa, u16 qid) 30364b9f64fSJason Wang { 30464b9f64fSJason Wang struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 30564b9f64fSJason Wang 30664b9f64fSJason Wang vp_iowrite16(qid, vp_vdpa->vring[qid].notify); 30764b9f64fSJason Wang } 30864b9f64fSJason Wang 30964b9f64fSJason Wang static u32 vp_vdpa_get_generation(struct vdpa_device *vdpa) 31064b9f64fSJason Wang { 31164b9f64fSJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 31264b9f64fSJason Wang 31364b9f64fSJason Wang return vp_modern_generation(mdev); 31464b9f64fSJason Wang } 31564b9f64fSJason Wang 31664b9f64fSJason Wang static u32 vp_vdpa_get_device_id(struct vdpa_device *vdpa) 31764b9f64fSJason Wang { 31864b9f64fSJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 31964b9f64fSJason Wang 32064b9f64fSJason Wang return mdev->id.device; 32164b9f64fSJason Wang } 32264b9f64fSJason Wang 32364b9f64fSJason Wang static u32 vp_vdpa_get_vendor_id(struct vdpa_device *vdpa) 32464b9f64fSJason Wang { 32564b9f64fSJason Wang struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 32664b9f64fSJason Wang 32764b9f64fSJason Wang return mdev->id.vendor; 32864b9f64fSJason Wang } 32964b9f64fSJason Wang 33064b9f64fSJason Wang static u32 vp_vdpa_get_vq_align(struct vdpa_device *vdpa) 33164b9f64fSJason Wang { 33264b9f64fSJason Wang return PAGE_SIZE; 33364b9f64fSJason Wang } 33464b9f64fSJason Wang 335442706f9SStefano Garzarella static size_t vp_vdpa_get_config_size(struct vdpa_device *vdpa) 336442706f9SStefano Garzarella { 337442706f9SStefano Garzarella struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 338442706f9SStefano Garzarella 339442706f9SStefano Garzarella return mdev->device_len; 340442706f9SStefano Garzarella } 341442706f9SStefano Garzarella 34264b9f64fSJason Wang static void vp_vdpa_get_config(struct vdpa_device *vdpa, 34364b9f64fSJason Wang unsigned int offset, 34464b9f64fSJason Wang void *buf, unsigned int len) 34564b9f64fSJason Wang { 34664b9f64fSJason Wang struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 34764b9f64fSJason Wang struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; 34864b9f64fSJason Wang u8 old, new; 34964b9f64fSJason Wang u8 *p; 35064b9f64fSJason Wang int i; 35164b9f64fSJason Wang 35264b9f64fSJason Wang do { 35364b9f64fSJason Wang old = vp_ioread8(&mdev->common->config_generation); 35464b9f64fSJason Wang p = buf; 35564b9f64fSJason Wang for (i = 0; i < len; i++) 35664b9f64fSJason Wang *p++ = vp_ioread8(mdev->device + offset + i); 35764b9f64fSJason Wang 35864b9f64fSJason Wang new = vp_ioread8(&mdev->common->config_generation); 35964b9f64fSJason Wang } while (old != new); 36064b9f64fSJason Wang } 36164b9f64fSJason Wang 36264b9f64fSJason Wang static void vp_vdpa_set_config(struct vdpa_device *vdpa, 36364b9f64fSJason Wang unsigned int offset, const void *buf, 36464b9f64fSJason Wang unsigned int len) 36564b9f64fSJason Wang { 36664b9f64fSJason Wang struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 36764b9f64fSJason Wang struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; 36864b9f64fSJason Wang const u8 *p = buf; 36964b9f64fSJason Wang int i; 37064b9f64fSJason Wang 37164b9f64fSJason Wang for (i = 0; i < len; i++) 37264b9f64fSJason Wang vp_iowrite8(*p++, mdev->device + offset + i); 37364b9f64fSJason Wang } 37464b9f64fSJason Wang 37564b9f64fSJason Wang static void vp_vdpa_set_config_cb(struct vdpa_device *vdpa, 37664b9f64fSJason Wang struct vdpa_callback *cb) 37764b9f64fSJason Wang { 37864b9f64fSJason Wang struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 37964b9f64fSJason Wang 38064b9f64fSJason Wang vp_vdpa->config_cb = *cb; 38164b9f64fSJason Wang } 38264b9f64fSJason Wang 383526cb858SJason Wang static struct vdpa_notification_area 384526cb858SJason Wang vp_vdpa_get_vq_notification(struct vdpa_device *vdpa, u16 qid) 385526cb858SJason Wang { 386526cb858SJason Wang struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 387526cb858SJason Wang struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; 388526cb858SJason Wang struct vdpa_notification_area notify; 389526cb858SJason Wang 390526cb858SJason Wang notify.addr = vp_vdpa->vring[qid].notify_pa; 391526cb858SJason Wang notify.size = mdev->notify_offset_multiplier; 392526cb858SJason Wang 393526cb858SJason Wang return notify; 394526cb858SJason Wang } 395526cb858SJason Wang 39664b9f64fSJason Wang static const struct vdpa_config_ops vp_vdpa_ops = { 39764b9f64fSJason Wang .get_features = vp_vdpa_get_features, 39864b9f64fSJason Wang .set_features = vp_vdpa_set_features, 39964b9f64fSJason Wang .get_status = vp_vdpa_get_status, 40064b9f64fSJason Wang .set_status = vp_vdpa_set_status, 40164b9f64fSJason Wang .get_vq_num_max = vp_vdpa_get_vq_num_max, 40264b9f64fSJason Wang .get_vq_state = vp_vdpa_get_vq_state, 403526cb858SJason Wang .get_vq_notification = vp_vdpa_get_vq_notification, 40464b9f64fSJason Wang .set_vq_state = vp_vdpa_set_vq_state, 40564b9f64fSJason Wang .set_vq_cb = vp_vdpa_set_vq_cb, 40664b9f64fSJason Wang .set_vq_ready = vp_vdpa_set_vq_ready, 40764b9f64fSJason Wang .get_vq_ready = vp_vdpa_get_vq_ready, 40864b9f64fSJason Wang .set_vq_num = vp_vdpa_set_vq_num, 40964b9f64fSJason Wang .set_vq_address = vp_vdpa_set_vq_address, 41064b9f64fSJason Wang .kick_vq = vp_vdpa_kick_vq, 41164b9f64fSJason Wang .get_generation = vp_vdpa_get_generation, 41264b9f64fSJason Wang .get_device_id = vp_vdpa_get_device_id, 41364b9f64fSJason Wang .get_vendor_id = vp_vdpa_get_vendor_id, 41464b9f64fSJason Wang .get_vq_align = vp_vdpa_get_vq_align, 415442706f9SStefano Garzarella .get_config_size = vp_vdpa_get_config_size, 41664b9f64fSJason Wang .get_config = vp_vdpa_get_config, 41764b9f64fSJason Wang .set_config = vp_vdpa_set_config, 41864b9f64fSJason Wang .set_config_cb = vp_vdpa_set_config_cb, 41964b9f64fSJason Wang }; 42064b9f64fSJason Wang 42164b9f64fSJason Wang static void vp_vdpa_free_irq_vectors(void *data) 42264b9f64fSJason Wang { 42364b9f64fSJason Wang pci_free_irq_vectors(data); 42464b9f64fSJason Wang } 42564b9f64fSJason Wang 42664b9f64fSJason Wang static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) 42764b9f64fSJason Wang { 42864b9f64fSJason Wang struct virtio_pci_modern_device *mdev; 42964b9f64fSJason Wang struct device *dev = &pdev->dev; 43064b9f64fSJason Wang struct vp_vdpa *vp_vdpa; 43164b9f64fSJason Wang int ret, i; 43264b9f64fSJason Wang 43364b9f64fSJason Wang ret = pcim_enable_device(pdev); 43464b9f64fSJason Wang if (ret) 43564b9f64fSJason Wang return ret; 43664b9f64fSJason Wang 43764b9f64fSJason Wang vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa, 43864b9f64fSJason Wang dev, &vp_vdpa_ops, NULL); 43964b9f64fSJason Wang if (vp_vdpa == NULL) { 44064b9f64fSJason Wang dev_err(dev, "vp_vdpa: Failed to allocate vDPA structure\n"); 44164b9f64fSJason Wang return -ENOMEM; 44264b9f64fSJason Wang } 44364b9f64fSJason Wang 44464b9f64fSJason Wang mdev = &vp_vdpa->mdev; 44564b9f64fSJason Wang mdev->pci_dev = pdev; 44664b9f64fSJason Wang 44764b9f64fSJason Wang ret = vp_modern_probe(mdev); 44864b9f64fSJason Wang if (ret) { 44964b9f64fSJason Wang dev_err(&pdev->dev, "Failed to probe modern PCI device\n"); 45064b9f64fSJason Wang goto err; 45164b9f64fSJason Wang } 45264b9f64fSJason Wang 45364b9f64fSJason Wang pci_set_master(pdev); 45464b9f64fSJason Wang pci_set_drvdata(pdev, vp_vdpa); 45564b9f64fSJason Wang 45664b9f64fSJason Wang vp_vdpa->vdpa.dma_dev = &pdev->dev; 45764b9f64fSJason Wang vp_vdpa->queues = vp_modern_get_num_queues(mdev); 45864b9f64fSJason Wang 45964b9f64fSJason Wang ret = devm_add_action_or_reset(dev, vp_vdpa_free_irq_vectors, pdev); 46064b9f64fSJason Wang if (ret) { 46164b9f64fSJason Wang dev_err(&pdev->dev, 46264b9f64fSJason Wang "Failed for adding devres for freeing irq vectors\n"); 46364b9f64fSJason Wang goto err; 46464b9f64fSJason Wang } 46564b9f64fSJason Wang 46664b9f64fSJason Wang vp_vdpa->vring = devm_kcalloc(&pdev->dev, vp_vdpa->queues, 46764b9f64fSJason Wang sizeof(*vp_vdpa->vring), 46864b9f64fSJason Wang GFP_KERNEL); 46964b9f64fSJason Wang if (!vp_vdpa->vring) { 47064b9f64fSJason Wang ret = -ENOMEM; 47164b9f64fSJason Wang dev_err(&pdev->dev, "Fail to allocate virtqueues\n"); 47264b9f64fSJason Wang goto err; 47364b9f64fSJason Wang } 47464b9f64fSJason Wang 47564b9f64fSJason Wang for (i = 0; i < vp_vdpa->queues; i++) { 47664b9f64fSJason Wang vp_vdpa->vring[i].irq = VIRTIO_MSI_NO_VECTOR; 4779e311bcaSJason Wang vp_vdpa->vring[i].notify = 478526cb858SJason Wang vp_modern_map_vq_notify(mdev, i, 479526cb858SJason Wang &vp_vdpa->vring[i].notify_pa); 48011d8ffedSJason Wang if (!vp_vdpa->vring[i].notify) { 48194e48d6aSJason Wang ret = -EINVAL; 48211d8ffedSJason Wang dev_warn(&pdev->dev, "Fail to map vq notify %d\n", i); 48311d8ffedSJason Wang goto err; 48411d8ffedSJason Wang } 48564b9f64fSJason Wang } 48664b9f64fSJason Wang vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR; 48764b9f64fSJason Wang 48864b9f64fSJason Wang ret = vdpa_register_device(&vp_vdpa->vdpa, vp_vdpa->queues); 48964b9f64fSJason Wang if (ret) { 49064b9f64fSJason Wang dev_err(&pdev->dev, "Failed to register to vdpa bus\n"); 49164b9f64fSJason Wang goto err; 49264b9f64fSJason Wang } 49364b9f64fSJason Wang 49464b9f64fSJason Wang return 0; 49564b9f64fSJason Wang 49664b9f64fSJason Wang err: 49764b9f64fSJason Wang put_device(&vp_vdpa->vdpa.dev); 49864b9f64fSJason Wang return ret; 49964b9f64fSJason Wang } 50064b9f64fSJason Wang 50164b9f64fSJason Wang static void vp_vdpa_remove(struct pci_dev *pdev) 50264b9f64fSJason Wang { 50364b9f64fSJason Wang struct vp_vdpa *vp_vdpa = pci_get_drvdata(pdev); 50464b9f64fSJason Wang 50564b9f64fSJason Wang vdpa_unregister_device(&vp_vdpa->vdpa); 50664b9f64fSJason Wang vp_modern_remove(&vp_vdpa->mdev); 50764b9f64fSJason Wang } 50864b9f64fSJason Wang 50964b9f64fSJason Wang static struct pci_driver vp_vdpa_driver = { 51064b9f64fSJason Wang .name = "vp-vdpa", 51164b9f64fSJason Wang .id_table = NULL, /* only dynamic ids */ 51264b9f64fSJason Wang .probe = vp_vdpa_probe, 51364b9f64fSJason Wang .remove = vp_vdpa_remove, 51464b9f64fSJason Wang }; 51564b9f64fSJason Wang 51664b9f64fSJason Wang module_pci_driver(vp_vdpa_driver); 51764b9f64fSJason Wang 51864b9f64fSJason Wang MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>"); 51964b9f64fSJason Wang MODULE_DESCRIPTION("vp-vdpa"); 52064b9f64fSJason Wang MODULE_LICENSE("GPL"); 52164b9f64fSJason Wang MODULE_VERSION("1"); 522