151a8f9d7SAlvaro Karsz // SPDX-License-Identifier: GPL-2.0-only
251a8f9d7SAlvaro Karsz /*
351a8f9d7SAlvaro Karsz * SolidRun DPU driver for control plane
451a8f9d7SAlvaro Karsz *
53f3a1675SAlvaro Karsz * Copyright (C) 2022-2023 SolidRun
651a8f9d7SAlvaro Karsz *
751a8f9d7SAlvaro Karsz * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
851a8f9d7SAlvaro Karsz *
951a8f9d7SAlvaro Karsz */
1051a8f9d7SAlvaro Karsz #include <linux/iopoll.h>
1151a8f9d7SAlvaro Karsz
1251a8f9d7SAlvaro Karsz #include "snet_vdpa.h"
1351a8f9d7SAlvaro Karsz
1451a8f9d7SAlvaro Karsz /* SNET DPU device ID */
1551a8f9d7SAlvaro Karsz #define SNET_DEVICE_ID 0x1000
1651a8f9d7SAlvaro Karsz /* SNET signature */
1751a8f9d7SAlvaro Karsz #define SNET_SIGNATURE 0xD0D06363
1851a8f9d7SAlvaro Karsz /* Max. config version that we can work with */
193f3a1675SAlvaro Karsz #define SNET_CFG_VERSION 0x2
2051a8f9d7SAlvaro Karsz /* Queue align */
2151a8f9d7SAlvaro Karsz #define SNET_QUEUE_ALIGNMENT PAGE_SIZE
2251a8f9d7SAlvaro Karsz /* Kick value to notify that new data is available */
2351a8f9d7SAlvaro Karsz #define SNET_KICK_VAL 0x1
2451a8f9d7SAlvaro Karsz #define SNET_CONFIG_OFF 0x0
2551a8f9d7SAlvaro Karsz /* How long we are willing to wait for a SNET device */
2651a8f9d7SAlvaro Karsz #define SNET_DETECT_TIMEOUT 5000000
2751a8f9d7SAlvaro Karsz /* How long should we wait for the DPU to read our config */
2851a8f9d7SAlvaro Karsz #define SNET_READ_CFG_TIMEOUT 3000000
2951a8f9d7SAlvaro Karsz /* Size of configs written to the DPU */
3051a8f9d7SAlvaro Karsz #define SNET_GENERAL_CFG_LEN 36
3151a8f9d7SAlvaro Karsz #define SNET_GENERAL_CFG_VQ_LEN 40
3251a8f9d7SAlvaro Karsz
vdpa_to_snet(struct vdpa_device * vdpa)3351a8f9d7SAlvaro Karsz static struct snet *vdpa_to_snet(struct vdpa_device *vdpa)
3451a8f9d7SAlvaro Karsz {
3551a8f9d7SAlvaro Karsz return container_of(vdpa, struct snet, vdpa);
3651a8f9d7SAlvaro Karsz }
3751a8f9d7SAlvaro Karsz
snet_cfg_irq_hndlr(int irq,void * data)3851a8f9d7SAlvaro Karsz static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data)
3951a8f9d7SAlvaro Karsz {
4051a8f9d7SAlvaro Karsz struct snet *snet = data;
4151a8f9d7SAlvaro Karsz /* Call callback if any */
425b250facSAlvaro Karsz if (likely(snet->cb.callback))
4351a8f9d7SAlvaro Karsz return snet->cb.callback(snet->cb.private);
4451a8f9d7SAlvaro Karsz
4551a8f9d7SAlvaro Karsz return IRQ_HANDLED;
4651a8f9d7SAlvaro Karsz }
4751a8f9d7SAlvaro Karsz
snet_vq_irq_hndlr(int irq,void * data)4851a8f9d7SAlvaro Karsz static irqreturn_t snet_vq_irq_hndlr(int irq, void *data)
4951a8f9d7SAlvaro Karsz {
5051a8f9d7SAlvaro Karsz struct snet_vq *vq = data;
5151a8f9d7SAlvaro Karsz /* Call callback if any */
525b250facSAlvaro Karsz if (likely(vq->cb.callback))
5351a8f9d7SAlvaro Karsz return vq->cb.callback(vq->cb.private);
5451a8f9d7SAlvaro Karsz
5551a8f9d7SAlvaro Karsz return IRQ_HANDLED;
5651a8f9d7SAlvaro Karsz }
5751a8f9d7SAlvaro Karsz
snet_free_irqs(struct snet * snet)5851a8f9d7SAlvaro Karsz static void snet_free_irqs(struct snet *snet)
5951a8f9d7SAlvaro Karsz {
6051a8f9d7SAlvaro Karsz struct psnet *psnet = snet->psnet;
6151a8f9d7SAlvaro Karsz struct pci_dev *pdev;
6251a8f9d7SAlvaro Karsz u32 i;
6351a8f9d7SAlvaro Karsz
6451a8f9d7SAlvaro Karsz /* Which Device allcoated the IRQs? */
6551a8f9d7SAlvaro Karsz if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))
6651a8f9d7SAlvaro Karsz pdev = snet->pdev->physfn;
6751a8f9d7SAlvaro Karsz else
6851a8f9d7SAlvaro Karsz pdev = snet->pdev;
6951a8f9d7SAlvaro Karsz
7051a8f9d7SAlvaro Karsz /* Free config's IRQ */
7151a8f9d7SAlvaro Karsz if (snet->cfg_irq != -1) {
7251a8f9d7SAlvaro Karsz devm_free_irq(&pdev->dev, snet->cfg_irq, snet);
7351a8f9d7SAlvaro Karsz snet->cfg_irq = -1;
7451a8f9d7SAlvaro Karsz }
7551a8f9d7SAlvaro Karsz /* Free VQ IRQs */
7651a8f9d7SAlvaro Karsz for (i = 0; i < snet->cfg->vq_num; i++) {
7751a8f9d7SAlvaro Karsz if (snet->vqs[i] && snet->vqs[i]->irq != -1) {
7851a8f9d7SAlvaro Karsz devm_free_irq(&pdev->dev, snet->vqs[i]->irq, snet->vqs[i]);
7951a8f9d7SAlvaro Karsz snet->vqs[i]->irq = -1;
8051a8f9d7SAlvaro Karsz }
8151a8f9d7SAlvaro Karsz }
8251a8f9d7SAlvaro Karsz
8351a8f9d7SAlvaro Karsz /* IRQ vectors are freed when the pci remove callback is called */
8451a8f9d7SAlvaro Karsz }
8551a8f9d7SAlvaro Karsz
snet_set_vq_address(struct vdpa_device * vdev,u16 idx,u64 desc_area,u64 driver_area,u64 device_area)8651a8f9d7SAlvaro Karsz static int snet_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area,
8751a8f9d7SAlvaro Karsz u64 driver_area, u64 device_area)
8851a8f9d7SAlvaro Karsz {
8951a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
9051a8f9d7SAlvaro Karsz /* save received parameters in vqueue sturct */
9151a8f9d7SAlvaro Karsz snet->vqs[idx]->desc_area = desc_area;
9251a8f9d7SAlvaro Karsz snet->vqs[idx]->driver_area = driver_area;
9351a8f9d7SAlvaro Karsz snet->vqs[idx]->device_area = device_area;
9451a8f9d7SAlvaro Karsz
9551a8f9d7SAlvaro Karsz return 0;
9651a8f9d7SAlvaro Karsz }
9751a8f9d7SAlvaro Karsz
snet_set_vq_num(struct vdpa_device * vdev,u16 idx,u32 num)9851a8f9d7SAlvaro Karsz static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num)
9951a8f9d7SAlvaro Karsz {
10051a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
10151a8f9d7SAlvaro Karsz /* save num in vqueue */
10251a8f9d7SAlvaro Karsz snet->vqs[idx]->num = num;
10351a8f9d7SAlvaro Karsz }
10451a8f9d7SAlvaro Karsz
snet_kick_vq(struct vdpa_device * vdev,u16 idx)10551a8f9d7SAlvaro Karsz static void snet_kick_vq(struct vdpa_device *vdev, u16 idx)
10651a8f9d7SAlvaro Karsz {
10751a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
10851a8f9d7SAlvaro Karsz /* not ready - ignore */
1095b250facSAlvaro Karsz if (unlikely(!snet->vqs[idx]->ready))
11051a8f9d7SAlvaro Karsz return;
11151a8f9d7SAlvaro Karsz
11251a8f9d7SAlvaro Karsz iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
11351a8f9d7SAlvaro Karsz }
11451a8f9d7SAlvaro Karsz
snet_kick_vq_with_data(struct vdpa_device * vdev,u32 data)11551b6e6c1SAlvaro Karsz static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data)
11651b6e6c1SAlvaro Karsz {
11751b6e6c1SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
11851b6e6c1SAlvaro Karsz u16 idx = data & 0xFFFF;
11951b6e6c1SAlvaro Karsz
12051b6e6c1SAlvaro Karsz /* not ready - ignore */
12151b6e6c1SAlvaro Karsz if (unlikely(!snet->vqs[idx]->ready))
12251b6e6c1SAlvaro Karsz return;
12351b6e6c1SAlvaro Karsz
12451b6e6c1SAlvaro Karsz iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);
12551b6e6c1SAlvaro Karsz }
12651b6e6c1SAlvaro Karsz
snet_set_vq_cb(struct vdpa_device * vdev,u16 idx,struct vdpa_callback * cb)12751a8f9d7SAlvaro Karsz static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb)
12851a8f9d7SAlvaro Karsz {
12951a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
13051a8f9d7SAlvaro Karsz
13151a8f9d7SAlvaro Karsz snet->vqs[idx]->cb.callback = cb->callback;
13251a8f9d7SAlvaro Karsz snet->vqs[idx]->cb.private = cb->private;
13351a8f9d7SAlvaro Karsz }
13451a8f9d7SAlvaro Karsz
snet_set_vq_ready(struct vdpa_device * vdev,u16 idx,bool ready)13551a8f9d7SAlvaro Karsz static void snet_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready)
13651a8f9d7SAlvaro Karsz {
13751a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
13851a8f9d7SAlvaro Karsz
13951a8f9d7SAlvaro Karsz snet->vqs[idx]->ready = ready;
14051a8f9d7SAlvaro Karsz }
14151a8f9d7SAlvaro Karsz
snet_get_vq_ready(struct vdpa_device * vdev,u16 idx)14251a8f9d7SAlvaro Karsz static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx)
14351a8f9d7SAlvaro Karsz {
14451a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
14551a8f9d7SAlvaro Karsz
14651a8f9d7SAlvaro Karsz return snet->vqs[idx]->ready;
14751a8f9d7SAlvaro Karsz }
14851a8f9d7SAlvaro Karsz
snet_vq_state_is_initial(struct snet * snet,const struct vdpa_vq_state * state)1493f3a1675SAlvaro Karsz static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state)
15051a8f9d7SAlvaro Karsz {
15151a8f9d7SAlvaro Karsz if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) {
15251a8f9d7SAlvaro Karsz const struct vdpa_vq_state_packed *p = &state->packed;
15351a8f9d7SAlvaro Karsz
15451a8f9d7SAlvaro Karsz if (p->last_avail_counter == 1 && p->last_used_counter == 1 &&
15551a8f9d7SAlvaro Karsz p->last_avail_idx == 0 && p->last_used_idx == 0)
1563f3a1675SAlvaro Karsz return true;
15751a8f9d7SAlvaro Karsz } else {
15851a8f9d7SAlvaro Karsz const struct vdpa_vq_state_split *s = &state->split;
15951a8f9d7SAlvaro Karsz
16051a8f9d7SAlvaro Karsz if (s->avail_index == 0)
1613f3a1675SAlvaro Karsz return true;
1623f3a1675SAlvaro Karsz }
1633f3a1675SAlvaro Karsz
1643f3a1675SAlvaro Karsz return false;
1653f3a1675SAlvaro Karsz }
1663f3a1675SAlvaro Karsz
snet_set_vq_state(struct vdpa_device * vdev,u16 idx,const struct vdpa_vq_state * state)1673f3a1675SAlvaro Karsz static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state)
1683f3a1675SAlvaro Karsz {
1693f3a1675SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
1703f3a1675SAlvaro Karsz
1713f3a1675SAlvaro Karsz /* We can set any state for config version 2+ */
1723f3a1675SAlvaro Karsz if (SNET_CFG_VER(snet, 2)) {
1733f3a1675SAlvaro Karsz memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state));
17451a8f9d7SAlvaro Karsz return 0;
17551a8f9d7SAlvaro Karsz }
17651a8f9d7SAlvaro Karsz
1773f3a1675SAlvaro Karsz /* Older config - we can't set the VQ state.
1783f3a1675SAlvaro Karsz * Return 0 only if this is the initial state we use in the DPU.
1793f3a1675SAlvaro Karsz */
1803f3a1675SAlvaro Karsz if (snet_vq_state_is_initial(snet, state))
1813f3a1675SAlvaro Karsz return 0;
1823f3a1675SAlvaro Karsz
18351a8f9d7SAlvaro Karsz return -EOPNOTSUPP;
18451a8f9d7SAlvaro Karsz }
18551a8f9d7SAlvaro Karsz
snet_get_vq_state(struct vdpa_device * vdev,u16 idx,struct vdpa_vq_state * state)18651a8f9d7SAlvaro Karsz static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state)
18751a8f9d7SAlvaro Karsz {
1883f3a1675SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
1893f3a1675SAlvaro Karsz
1903f3a1675SAlvaro Karsz return snet_read_vq_state(snet, idx, state);
19151a8f9d7SAlvaro Karsz }
19251a8f9d7SAlvaro Karsz
snet_get_vq_irq(struct vdpa_device * vdev,u16 idx)19351a8f9d7SAlvaro Karsz static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx)
19451a8f9d7SAlvaro Karsz {
19551a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
19651a8f9d7SAlvaro Karsz
19751a8f9d7SAlvaro Karsz return snet->vqs[idx]->irq;
19851a8f9d7SAlvaro Karsz }
19951a8f9d7SAlvaro Karsz
snet_get_vq_align(struct vdpa_device * vdev)20051a8f9d7SAlvaro Karsz static u32 snet_get_vq_align(struct vdpa_device *vdev)
20151a8f9d7SAlvaro Karsz {
20251a8f9d7SAlvaro Karsz return (u32)SNET_QUEUE_ALIGNMENT;
20351a8f9d7SAlvaro Karsz }
20451a8f9d7SAlvaro Karsz
snet_reset_dev(struct snet * snet)20551a8f9d7SAlvaro Karsz static int snet_reset_dev(struct snet *snet)
20651a8f9d7SAlvaro Karsz {
20751a8f9d7SAlvaro Karsz struct pci_dev *pdev = snet->pdev;
20851a8f9d7SAlvaro Karsz int ret = 0;
20951a8f9d7SAlvaro Karsz u32 i;
21051a8f9d7SAlvaro Karsz
21151a8f9d7SAlvaro Karsz /* If status is 0, nothing to do */
21251a8f9d7SAlvaro Karsz if (!snet->status)
21351a8f9d7SAlvaro Karsz return 0;
21451a8f9d7SAlvaro Karsz
2153f3a1675SAlvaro Karsz /* If DPU started, destroy it */
21651a8f9d7SAlvaro Karsz if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK)
2173f3a1675SAlvaro Karsz ret = snet_destroy_dev(snet);
21851a8f9d7SAlvaro Karsz
21951a8f9d7SAlvaro Karsz /* Clear VQs */
22051a8f9d7SAlvaro Karsz for (i = 0; i < snet->cfg->vq_num; i++) {
22151a8f9d7SAlvaro Karsz if (!snet->vqs[i])
22251a8f9d7SAlvaro Karsz continue;
22351a8f9d7SAlvaro Karsz snet->vqs[i]->cb.callback = NULL;
22451a8f9d7SAlvaro Karsz snet->vqs[i]->cb.private = NULL;
22551a8f9d7SAlvaro Karsz snet->vqs[i]->desc_area = 0;
22651a8f9d7SAlvaro Karsz snet->vqs[i]->device_area = 0;
22751a8f9d7SAlvaro Karsz snet->vqs[i]->driver_area = 0;
22851a8f9d7SAlvaro Karsz snet->vqs[i]->ready = false;
22951a8f9d7SAlvaro Karsz }
23051a8f9d7SAlvaro Karsz
23151a8f9d7SAlvaro Karsz /* Clear config callback */
23251a8f9d7SAlvaro Karsz snet->cb.callback = NULL;
23351a8f9d7SAlvaro Karsz snet->cb.private = NULL;
23451a8f9d7SAlvaro Karsz /* Free IRQs */
23551a8f9d7SAlvaro Karsz snet_free_irqs(snet);
23651a8f9d7SAlvaro Karsz /* Reset status */
23751a8f9d7SAlvaro Karsz snet->status = 0;
23851a8f9d7SAlvaro Karsz snet->dpu_ready = false;
23951a8f9d7SAlvaro Karsz
24051a8f9d7SAlvaro Karsz if (ret)
2413f3a1675SAlvaro Karsz SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret);
24251a8f9d7SAlvaro Karsz else
24351a8f9d7SAlvaro Karsz SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid);
24451a8f9d7SAlvaro Karsz
24551a8f9d7SAlvaro Karsz return 0;
24651a8f9d7SAlvaro Karsz }
24751a8f9d7SAlvaro Karsz
snet_reset(struct vdpa_device * vdev)24851a8f9d7SAlvaro Karsz static int snet_reset(struct vdpa_device *vdev)
24951a8f9d7SAlvaro Karsz {
25051a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
25151a8f9d7SAlvaro Karsz
25251a8f9d7SAlvaro Karsz return snet_reset_dev(snet);
25351a8f9d7SAlvaro Karsz }
25451a8f9d7SAlvaro Karsz
snet_get_config_size(struct vdpa_device * vdev)25551a8f9d7SAlvaro Karsz static size_t snet_get_config_size(struct vdpa_device *vdev)
25651a8f9d7SAlvaro Karsz {
25751a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
25851a8f9d7SAlvaro Karsz
25951a8f9d7SAlvaro Karsz return (size_t)snet->cfg->cfg_size;
26051a8f9d7SAlvaro Karsz }
26151a8f9d7SAlvaro Karsz
snet_get_features(struct vdpa_device * vdev)26251a8f9d7SAlvaro Karsz static u64 snet_get_features(struct vdpa_device *vdev)
26351a8f9d7SAlvaro Karsz {
26451a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
26551a8f9d7SAlvaro Karsz
26651a8f9d7SAlvaro Karsz return snet->cfg->features;
26751a8f9d7SAlvaro Karsz }
26851a8f9d7SAlvaro Karsz
snet_set_drv_features(struct vdpa_device * vdev,u64 features)26951a8f9d7SAlvaro Karsz static int snet_set_drv_features(struct vdpa_device *vdev, u64 features)
27051a8f9d7SAlvaro Karsz {
27151a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
27251a8f9d7SAlvaro Karsz
27351a8f9d7SAlvaro Karsz snet->negotiated_features = snet->cfg->features & features;
27451a8f9d7SAlvaro Karsz return 0;
27551a8f9d7SAlvaro Karsz }
27651a8f9d7SAlvaro Karsz
snet_get_drv_features(struct vdpa_device * vdev)27751a8f9d7SAlvaro Karsz static u64 snet_get_drv_features(struct vdpa_device *vdev)
27851a8f9d7SAlvaro Karsz {
27951a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
28051a8f9d7SAlvaro Karsz
28151a8f9d7SAlvaro Karsz return snet->negotiated_features;
28251a8f9d7SAlvaro Karsz }
28351a8f9d7SAlvaro Karsz
snet_get_vq_num_max(struct vdpa_device * vdev)28451a8f9d7SAlvaro Karsz static u16 snet_get_vq_num_max(struct vdpa_device *vdev)
28551a8f9d7SAlvaro Karsz {
28651a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
28751a8f9d7SAlvaro Karsz
28851a8f9d7SAlvaro Karsz return (u16)snet->cfg->vq_size;
28951a8f9d7SAlvaro Karsz }
29051a8f9d7SAlvaro Karsz
snet_set_config_cb(struct vdpa_device * vdev,struct vdpa_callback * cb)29151a8f9d7SAlvaro Karsz static void snet_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb)
29251a8f9d7SAlvaro Karsz {
29351a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
29451a8f9d7SAlvaro Karsz
29551a8f9d7SAlvaro Karsz snet->cb.callback = cb->callback;
29651a8f9d7SAlvaro Karsz snet->cb.private = cb->private;
29751a8f9d7SAlvaro Karsz }
29851a8f9d7SAlvaro Karsz
snet_get_device_id(struct vdpa_device * vdev)29951a8f9d7SAlvaro Karsz static u32 snet_get_device_id(struct vdpa_device *vdev)
30051a8f9d7SAlvaro Karsz {
30151a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
30251a8f9d7SAlvaro Karsz
30351a8f9d7SAlvaro Karsz return snet->cfg->virtio_id;
30451a8f9d7SAlvaro Karsz }
30551a8f9d7SAlvaro Karsz
snet_get_vendor_id(struct vdpa_device * vdev)30651a8f9d7SAlvaro Karsz static u32 snet_get_vendor_id(struct vdpa_device *vdev)
30751a8f9d7SAlvaro Karsz {
30851a8f9d7SAlvaro Karsz return (u32)PCI_VENDOR_ID_SOLIDRUN;
30951a8f9d7SAlvaro Karsz }
31051a8f9d7SAlvaro Karsz
snet_get_status(struct vdpa_device * vdev)31151a8f9d7SAlvaro Karsz static u8 snet_get_status(struct vdpa_device *vdev)
31251a8f9d7SAlvaro Karsz {
31351a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
31451a8f9d7SAlvaro Karsz
31551a8f9d7SAlvaro Karsz return snet->status;
31651a8f9d7SAlvaro Karsz }
31751a8f9d7SAlvaro Karsz
snet_write_conf(struct snet * snet)31851a8f9d7SAlvaro Karsz static int snet_write_conf(struct snet *snet)
31951a8f9d7SAlvaro Karsz {
32051a8f9d7SAlvaro Karsz u32 off, i, tmp;
32151a8f9d7SAlvaro Karsz int ret;
32251a8f9d7SAlvaro Karsz
32351a8f9d7SAlvaro Karsz /* No need to write the config twice */
32451a8f9d7SAlvaro Karsz if (snet->dpu_ready)
32551a8f9d7SAlvaro Karsz return true;
32651a8f9d7SAlvaro Karsz
32751a8f9d7SAlvaro Karsz /* Snet data :
32851a8f9d7SAlvaro Karsz *
32951a8f9d7SAlvaro Karsz * General data: SNET_GENERAL_CFG_LEN bytes long
33051a8f9d7SAlvaro Karsz * 0 0x4 0x8 0xC 0x10 0x14 0x1C 0x24
33151a8f9d7SAlvaro Karsz * | MAGIC NUMBER | CFG VER | SNET SID | NUMBER OF QUEUES | IRQ IDX | FEATURES | RSVD |
33251a8f9d7SAlvaro Karsz *
33351a8f9d7SAlvaro Karsz * For every VQ: SNET_GENERAL_CFG_VQ_LEN bytes long
33451a8f9d7SAlvaro Karsz * 0 0x4 0x8
33551a8f9d7SAlvaro Karsz * | VQ SID AND QUEUE SIZE | IRQ Index |
33651a8f9d7SAlvaro Karsz * | DESC AREA |
33751a8f9d7SAlvaro Karsz * | DEVICE AREA |
33851a8f9d7SAlvaro Karsz * | DRIVER AREA |
3393f3a1675SAlvaro Karsz * | VQ STATE (CFG 2+) | RSVD |
34051a8f9d7SAlvaro Karsz *
34151a8f9d7SAlvaro Karsz * Magic number should be written last, this is the DPU indication that the data is ready
34251a8f9d7SAlvaro Karsz */
34351a8f9d7SAlvaro Karsz
34451a8f9d7SAlvaro Karsz /* Init offset */
34551a8f9d7SAlvaro Karsz off = snet->psnet->cfg.host_cfg_off;
34651a8f9d7SAlvaro Karsz
34751a8f9d7SAlvaro Karsz /* Ignore magic number for now */
34851a8f9d7SAlvaro Karsz off += 4;
34951a8f9d7SAlvaro Karsz snet_write32(snet, off, snet->psnet->negotiated_cfg_ver);
35051a8f9d7SAlvaro Karsz off += 4;
35151a8f9d7SAlvaro Karsz snet_write32(snet, off, snet->sid);
35251a8f9d7SAlvaro Karsz off += 4;
35351a8f9d7SAlvaro Karsz snet_write32(snet, off, snet->cfg->vq_num);
35451a8f9d7SAlvaro Karsz off += 4;
35551a8f9d7SAlvaro Karsz snet_write32(snet, off, snet->cfg_irq_idx);
35651a8f9d7SAlvaro Karsz off += 4;
35751a8f9d7SAlvaro Karsz snet_write64(snet, off, snet->negotiated_features);
35851a8f9d7SAlvaro Karsz off += 8;
35951a8f9d7SAlvaro Karsz /* Ignore reserved */
36051a8f9d7SAlvaro Karsz off += 8;
36151a8f9d7SAlvaro Karsz /* Write VQs */
36251a8f9d7SAlvaro Karsz for (i = 0 ; i < snet->cfg->vq_num ; i++) {
36351a8f9d7SAlvaro Karsz tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF);
36451a8f9d7SAlvaro Karsz snet_write32(snet, off, tmp);
36551a8f9d7SAlvaro Karsz off += 4;
36651a8f9d7SAlvaro Karsz snet_write32(snet, off, snet->vqs[i]->irq_idx);
36751a8f9d7SAlvaro Karsz off += 4;
36851a8f9d7SAlvaro Karsz snet_write64(snet, off, snet->vqs[i]->desc_area);
36951a8f9d7SAlvaro Karsz off += 8;
37051a8f9d7SAlvaro Karsz snet_write64(snet, off, snet->vqs[i]->device_area);
37151a8f9d7SAlvaro Karsz off += 8;
37251a8f9d7SAlvaro Karsz snet_write64(snet, off, snet->vqs[i]->driver_area);
37351a8f9d7SAlvaro Karsz off += 8;
3743f3a1675SAlvaro Karsz /* Write VQ state if config version is 2+ */
3753f3a1675SAlvaro Karsz if (SNET_CFG_VER(snet, 2))
3763f3a1675SAlvaro Karsz snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state);
3773f3a1675SAlvaro Karsz off += 4;
3783f3a1675SAlvaro Karsz
37951a8f9d7SAlvaro Karsz /* Ignore reserved */
3803f3a1675SAlvaro Karsz off += 4;
38151a8f9d7SAlvaro Karsz }
38251a8f9d7SAlvaro Karsz
38351a8f9d7SAlvaro Karsz /* Write magic number - data is ready */
38451a8f9d7SAlvaro Karsz snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE);
38551a8f9d7SAlvaro Karsz
38651a8f9d7SAlvaro Karsz /* The DPU will ACK the config by clearing the signature */
38751a8f9d7SAlvaro Karsz ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off,
38851a8f9d7SAlvaro Karsz tmp, !tmp, 10, SNET_READ_CFG_TIMEOUT);
38951a8f9d7SAlvaro Karsz if (ret) {
39051a8f9d7SAlvaro Karsz SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n");
39151a8f9d7SAlvaro Karsz return false;
39251a8f9d7SAlvaro Karsz }
39351a8f9d7SAlvaro Karsz
39451a8f9d7SAlvaro Karsz /* set DPU flag */
39551a8f9d7SAlvaro Karsz snet->dpu_ready = true;
39651a8f9d7SAlvaro Karsz
39751a8f9d7SAlvaro Karsz return true;
39851a8f9d7SAlvaro Karsz }
39951a8f9d7SAlvaro Karsz
snet_request_irqs(struct pci_dev * pdev,struct snet * snet)40051a8f9d7SAlvaro Karsz static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet)
40151a8f9d7SAlvaro Karsz {
40251a8f9d7SAlvaro Karsz int ret, i, irq;
40351a8f9d7SAlvaro Karsz
40451a8f9d7SAlvaro Karsz /* Request config IRQ */
40551a8f9d7SAlvaro Karsz irq = pci_irq_vector(pdev, snet->cfg_irq_idx);
40651a8f9d7SAlvaro Karsz ret = devm_request_irq(&pdev->dev, irq, snet_cfg_irq_hndlr, 0,
40751a8f9d7SAlvaro Karsz snet->cfg_irq_name, snet);
40851a8f9d7SAlvaro Karsz if (ret) {
40951a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to request IRQ\n");
41051a8f9d7SAlvaro Karsz return ret;
41151a8f9d7SAlvaro Karsz }
41251a8f9d7SAlvaro Karsz snet->cfg_irq = irq;
41351a8f9d7SAlvaro Karsz
41451a8f9d7SAlvaro Karsz /* Request IRQ for every VQ */
41551a8f9d7SAlvaro Karsz for (i = 0; i < snet->cfg->vq_num; i++) {
41651a8f9d7SAlvaro Karsz irq = pci_irq_vector(pdev, snet->vqs[i]->irq_idx);
41751a8f9d7SAlvaro Karsz ret = devm_request_irq(&pdev->dev, irq, snet_vq_irq_hndlr, 0,
41851a8f9d7SAlvaro Karsz snet->vqs[i]->irq_name, snet->vqs[i]);
41951a8f9d7SAlvaro Karsz if (ret) {
42051a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to request IRQ\n");
42151a8f9d7SAlvaro Karsz return ret;
42251a8f9d7SAlvaro Karsz }
42351a8f9d7SAlvaro Karsz snet->vqs[i]->irq = irq;
42451a8f9d7SAlvaro Karsz }
42551a8f9d7SAlvaro Karsz return 0;
42651a8f9d7SAlvaro Karsz }
42751a8f9d7SAlvaro Karsz
snet_set_status(struct vdpa_device * vdev,u8 status)42851a8f9d7SAlvaro Karsz static void snet_set_status(struct vdpa_device *vdev, u8 status)
42951a8f9d7SAlvaro Karsz {
43051a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
43151a8f9d7SAlvaro Karsz struct psnet *psnet = snet->psnet;
43251a8f9d7SAlvaro Karsz struct pci_dev *pdev = snet->pdev;
43351a8f9d7SAlvaro Karsz int ret;
43451a8f9d7SAlvaro Karsz bool pf_irqs;
43551a8f9d7SAlvaro Karsz
43651a8f9d7SAlvaro Karsz if (status == snet->status)
43751a8f9d7SAlvaro Karsz return;
43851a8f9d7SAlvaro Karsz
43951a8f9d7SAlvaro Karsz if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
44051a8f9d7SAlvaro Karsz !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
44151a8f9d7SAlvaro Karsz /* Request IRQs */
44251a8f9d7SAlvaro Karsz pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);
44351a8f9d7SAlvaro Karsz ret = snet_request_irqs(pf_irqs ? pdev->physfn : pdev, snet);
44451a8f9d7SAlvaro Karsz if (ret)
44551a8f9d7SAlvaro Karsz goto set_err;
44651a8f9d7SAlvaro Karsz
44751a8f9d7SAlvaro Karsz /* Write config to the DPU */
44851a8f9d7SAlvaro Karsz if (snet_write_conf(snet)) {
44951a8f9d7SAlvaro Karsz SNET_INFO(pdev, "Create SNET[%u] device\n", snet->sid);
45051a8f9d7SAlvaro Karsz } else {
45151a8f9d7SAlvaro Karsz snet_free_irqs(snet);
45251a8f9d7SAlvaro Karsz goto set_err;
45351a8f9d7SAlvaro Karsz }
45451a8f9d7SAlvaro Karsz }
45551a8f9d7SAlvaro Karsz
45651a8f9d7SAlvaro Karsz /* Save the new status */
45751a8f9d7SAlvaro Karsz snet->status = status;
45851a8f9d7SAlvaro Karsz return;
45951a8f9d7SAlvaro Karsz
46051a8f9d7SAlvaro Karsz set_err:
46151a8f9d7SAlvaro Karsz snet->status |= VIRTIO_CONFIG_S_FAILED;
46251a8f9d7SAlvaro Karsz }
46351a8f9d7SAlvaro Karsz
snet_get_config(struct vdpa_device * vdev,unsigned int offset,void * buf,unsigned int len)46451a8f9d7SAlvaro Karsz static void snet_get_config(struct vdpa_device *vdev, unsigned int offset,
46551a8f9d7SAlvaro Karsz void *buf, unsigned int len)
46651a8f9d7SAlvaro Karsz {
46751a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
46851a8f9d7SAlvaro Karsz void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset;
46951a8f9d7SAlvaro Karsz u8 *buf_ptr = buf;
47051a8f9d7SAlvaro Karsz u32 i;
47151a8f9d7SAlvaro Karsz
47251a8f9d7SAlvaro Karsz /* check for offset error */
47351a8f9d7SAlvaro Karsz if (offset + len > snet->cfg->cfg_size)
47451a8f9d7SAlvaro Karsz return;
47551a8f9d7SAlvaro Karsz
47651a8f9d7SAlvaro Karsz /* Write into buffer */
47751a8f9d7SAlvaro Karsz for (i = 0; i < len; i++)
47851a8f9d7SAlvaro Karsz *buf_ptr++ = ioread8(cfg_ptr + i);
47951a8f9d7SAlvaro Karsz }
48051a8f9d7SAlvaro Karsz
snet_set_config(struct vdpa_device * vdev,unsigned int offset,const void * buf,unsigned int len)48151a8f9d7SAlvaro Karsz static void snet_set_config(struct vdpa_device *vdev, unsigned int offset,
48251a8f9d7SAlvaro Karsz const void *buf, unsigned int len)
48351a8f9d7SAlvaro Karsz {
48451a8f9d7SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
48551a8f9d7SAlvaro Karsz void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset;
48651a8f9d7SAlvaro Karsz const u8 *buf_ptr = buf;
48751a8f9d7SAlvaro Karsz u32 i;
48851a8f9d7SAlvaro Karsz
48951a8f9d7SAlvaro Karsz /* check for offset error */
49051a8f9d7SAlvaro Karsz if (offset + len > snet->cfg->cfg_size)
49151a8f9d7SAlvaro Karsz return;
49251a8f9d7SAlvaro Karsz
49351a8f9d7SAlvaro Karsz /* Write into PCI BAR */
49451a8f9d7SAlvaro Karsz for (i = 0; i < len; i++)
49551a8f9d7SAlvaro Karsz iowrite8(*buf_ptr++, cfg_ptr + i);
49651a8f9d7SAlvaro Karsz }
49751a8f9d7SAlvaro Karsz
snet_suspend(struct vdpa_device * vdev)4983616bf37SAlvaro Karsz static int snet_suspend(struct vdpa_device *vdev)
4993616bf37SAlvaro Karsz {
5003616bf37SAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
5013616bf37SAlvaro Karsz int ret;
5023616bf37SAlvaro Karsz
5033616bf37SAlvaro Karsz ret = snet_suspend_dev(snet);
5043616bf37SAlvaro Karsz if (ret)
5053616bf37SAlvaro Karsz SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret);
5063616bf37SAlvaro Karsz else
5073616bf37SAlvaro Karsz SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid);
5083616bf37SAlvaro Karsz
5093616bf37SAlvaro Karsz return ret;
5103616bf37SAlvaro Karsz }
5113616bf37SAlvaro Karsz
snet_resume(struct vdpa_device * vdev)512*ef9da01cSAlvaro Karsz static int snet_resume(struct vdpa_device *vdev)
513*ef9da01cSAlvaro Karsz {
514*ef9da01cSAlvaro Karsz struct snet *snet = vdpa_to_snet(vdev);
515*ef9da01cSAlvaro Karsz int ret;
516*ef9da01cSAlvaro Karsz
517*ef9da01cSAlvaro Karsz ret = snet_resume_dev(snet);
518*ef9da01cSAlvaro Karsz if (ret)
519*ef9da01cSAlvaro Karsz SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n", snet->sid, ret);
520*ef9da01cSAlvaro Karsz else
521*ef9da01cSAlvaro Karsz SNET_DBG(snet->pdev, "Resume SNET[%u] device\n", snet->sid);
522*ef9da01cSAlvaro Karsz
523*ef9da01cSAlvaro Karsz return ret;
524*ef9da01cSAlvaro Karsz }
525*ef9da01cSAlvaro Karsz
52651a8f9d7SAlvaro Karsz static const struct vdpa_config_ops snet_config_ops = {
52751a8f9d7SAlvaro Karsz .set_vq_address = snet_set_vq_address,
52851a8f9d7SAlvaro Karsz .set_vq_num = snet_set_vq_num,
52951a8f9d7SAlvaro Karsz .kick_vq = snet_kick_vq,
53051b6e6c1SAlvaro Karsz .kick_vq_with_data = snet_kick_vq_with_data,
53151a8f9d7SAlvaro Karsz .set_vq_cb = snet_set_vq_cb,
53251a8f9d7SAlvaro Karsz .set_vq_ready = snet_set_vq_ready,
53351a8f9d7SAlvaro Karsz .get_vq_ready = snet_get_vq_ready,
53451a8f9d7SAlvaro Karsz .set_vq_state = snet_set_vq_state,
53551a8f9d7SAlvaro Karsz .get_vq_state = snet_get_vq_state,
53651a8f9d7SAlvaro Karsz .get_vq_irq = snet_get_vq_irq,
53751a8f9d7SAlvaro Karsz .get_vq_align = snet_get_vq_align,
53851a8f9d7SAlvaro Karsz .reset = snet_reset,
53951a8f9d7SAlvaro Karsz .get_config_size = snet_get_config_size,
54051a8f9d7SAlvaro Karsz .get_device_features = snet_get_features,
54151a8f9d7SAlvaro Karsz .set_driver_features = snet_set_drv_features,
54251a8f9d7SAlvaro Karsz .get_driver_features = snet_get_drv_features,
54351a8f9d7SAlvaro Karsz .get_vq_num_min = snet_get_vq_num_max,
54451a8f9d7SAlvaro Karsz .get_vq_num_max = snet_get_vq_num_max,
54551a8f9d7SAlvaro Karsz .set_config_cb = snet_set_config_cb,
54651a8f9d7SAlvaro Karsz .get_device_id = snet_get_device_id,
54751a8f9d7SAlvaro Karsz .get_vendor_id = snet_get_vendor_id,
54851a8f9d7SAlvaro Karsz .get_status = snet_get_status,
54951a8f9d7SAlvaro Karsz .set_status = snet_set_status,
55051a8f9d7SAlvaro Karsz .get_config = snet_get_config,
55151a8f9d7SAlvaro Karsz .set_config = snet_set_config,
5523616bf37SAlvaro Karsz .suspend = snet_suspend,
553*ef9da01cSAlvaro Karsz .resume = snet_resume,
55451a8f9d7SAlvaro Karsz };
55551a8f9d7SAlvaro Karsz
psnet_open_pf_bar(struct pci_dev * pdev,struct psnet * psnet)55651a8f9d7SAlvaro Karsz static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet)
55751a8f9d7SAlvaro Karsz {
55851a8f9d7SAlvaro Karsz char name[50];
55951a8f9d7SAlvaro Karsz int ret, i, mask = 0;
56051a8f9d7SAlvaro Karsz /* We don't know which BAR will be used to communicate..
56151a8f9d7SAlvaro Karsz * We will map every bar with len > 0.
56251a8f9d7SAlvaro Karsz *
56351a8f9d7SAlvaro Karsz * Later, we will discover the BAR and unmap all other BARs.
56451a8f9d7SAlvaro Karsz */
56551a8f9d7SAlvaro Karsz for (i = 0; i < PCI_STD_NUM_BARS; i++) {
56651a8f9d7SAlvaro Karsz if (pci_resource_len(pdev, i))
56751a8f9d7SAlvaro Karsz mask |= (1 << i);
56851a8f9d7SAlvaro Karsz }
56951a8f9d7SAlvaro Karsz
57051a8f9d7SAlvaro Karsz /* No BAR can be used.. */
57151a8f9d7SAlvaro Karsz if (!mask) {
57251a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to find a PCI BAR\n");
57351a8f9d7SAlvaro Karsz return -ENODEV;
57451a8f9d7SAlvaro Karsz }
57551a8f9d7SAlvaro Karsz
57651a8f9d7SAlvaro Karsz snprintf(name, sizeof(name), "psnet[%s]-bars", pci_name(pdev));
57751a8f9d7SAlvaro Karsz ret = pcim_iomap_regions(pdev, mask, name);
57851a8f9d7SAlvaro Karsz if (ret) {
57951a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to request and map PCI BARs\n");
58051a8f9d7SAlvaro Karsz return ret;
58151a8f9d7SAlvaro Karsz }
58251a8f9d7SAlvaro Karsz
58351a8f9d7SAlvaro Karsz for (i = 0; i < PCI_STD_NUM_BARS; i++) {
58451a8f9d7SAlvaro Karsz if (mask & (1 << i))
58551a8f9d7SAlvaro Karsz psnet->bars[i] = pcim_iomap_table(pdev)[i];
58651a8f9d7SAlvaro Karsz }
58751a8f9d7SAlvaro Karsz
58851a8f9d7SAlvaro Karsz return 0;
58951a8f9d7SAlvaro Karsz }
59051a8f9d7SAlvaro Karsz
snet_open_vf_bar(struct pci_dev * pdev,struct snet * snet)59151a8f9d7SAlvaro Karsz static int snet_open_vf_bar(struct pci_dev *pdev, struct snet *snet)
59251a8f9d7SAlvaro Karsz {
59351a8f9d7SAlvaro Karsz char name[50];
59451a8f9d7SAlvaro Karsz int ret;
59551a8f9d7SAlvaro Karsz
59651a8f9d7SAlvaro Karsz snprintf(name, sizeof(name), "snet[%s]-bar", pci_name(pdev));
59751a8f9d7SAlvaro Karsz /* Request and map BAR */
59851a8f9d7SAlvaro Karsz ret = pcim_iomap_regions(pdev, BIT(snet->psnet->cfg.vf_bar), name);
59951a8f9d7SAlvaro Karsz if (ret) {
60051a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to request and map PCI BAR for a VF\n");
60151a8f9d7SAlvaro Karsz return ret;
60251a8f9d7SAlvaro Karsz }
60351a8f9d7SAlvaro Karsz
60451a8f9d7SAlvaro Karsz snet->bar = pcim_iomap_table(pdev)[snet->psnet->cfg.vf_bar];
60551a8f9d7SAlvaro Karsz
60651a8f9d7SAlvaro Karsz return 0;
60751a8f9d7SAlvaro Karsz }
60851a8f9d7SAlvaro Karsz
snet_free_cfg(struct snet_cfg * cfg)60951a8f9d7SAlvaro Karsz static void snet_free_cfg(struct snet_cfg *cfg)
61051a8f9d7SAlvaro Karsz {
61151a8f9d7SAlvaro Karsz u32 i;
61251a8f9d7SAlvaro Karsz
61351a8f9d7SAlvaro Karsz if (!cfg->devs)
61451a8f9d7SAlvaro Karsz return;
61551a8f9d7SAlvaro Karsz
61651a8f9d7SAlvaro Karsz /* Free devices */
61751a8f9d7SAlvaro Karsz for (i = 0; i < cfg->devices_num; i++) {
61851a8f9d7SAlvaro Karsz if (!cfg->devs[i])
61951a8f9d7SAlvaro Karsz break;
62051a8f9d7SAlvaro Karsz
62151a8f9d7SAlvaro Karsz kfree(cfg->devs[i]);
62251a8f9d7SAlvaro Karsz }
62351a8f9d7SAlvaro Karsz /* Free pointers to devices */
62451a8f9d7SAlvaro Karsz kfree(cfg->devs);
62551a8f9d7SAlvaro Karsz }
62651a8f9d7SAlvaro Karsz
62751a8f9d7SAlvaro Karsz /* Detect which BAR is used for communication with the device. */
psnet_detect_bar(struct psnet * psnet,u32 off)62851a8f9d7SAlvaro Karsz static int psnet_detect_bar(struct psnet *psnet, u32 off)
62951a8f9d7SAlvaro Karsz {
63051a8f9d7SAlvaro Karsz unsigned long exit_time;
63151a8f9d7SAlvaro Karsz int i;
63251a8f9d7SAlvaro Karsz
63351a8f9d7SAlvaro Karsz exit_time = jiffies + usecs_to_jiffies(SNET_DETECT_TIMEOUT);
63451a8f9d7SAlvaro Karsz
63551a8f9d7SAlvaro Karsz /* SNET DPU will write SNET's signature when the config is ready. */
63651a8f9d7SAlvaro Karsz while (time_before(jiffies, exit_time)) {
63751a8f9d7SAlvaro Karsz for (i = 0; i < PCI_STD_NUM_BARS; i++) {
63851a8f9d7SAlvaro Karsz /* Is this BAR mapped? */
63951a8f9d7SAlvaro Karsz if (!psnet->bars[i])
64051a8f9d7SAlvaro Karsz continue;
64151a8f9d7SAlvaro Karsz
64251a8f9d7SAlvaro Karsz if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE)
64351a8f9d7SAlvaro Karsz return i;
64451a8f9d7SAlvaro Karsz }
64551a8f9d7SAlvaro Karsz usleep_range(1000, 10000);
64651a8f9d7SAlvaro Karsz }
64751a8f9d7SAlvaro Karsz
64851a8f9d7SAlvaro Karsz return -ENODEV;
64951a8f9d7SAlvaro Karsz }
65051a8f9d7SAlvaro Karsz
psnet_unmap_unused_bars(struct pci_dev * pdev,struct psnet * psnet)65151a8f9d7SAlvaro Karsz static void psnet_unmap_unused_bars(struct pci_dev *pdev, struct psnet *psnet)
65251a8f9d7SAlvaro Karsz {
65351a8f9d7SAlvaro Karsz int i, mask = 0;
65451a8f9d7SAlvaro Karsz
65551a8f9d7SAlvaro Karsz for (i = 0; i < PCI_STD_NUM_BARS; i++) {
65651a8f9d7SAlvaro Karsz if (psnet->bars[i] && i != psnet->barno)
65751a8f9d7SAlvaro Karsz mask |= (1 << i);
65851a8f9d7SAlvaro Karsz }
65951a8f9d7SAlvaro Karsz
66051a8f9d7SAlvaro Karsz if (mask)
66151a8f9d7SAlvaro Karsz pcim_iounmap_regions(pdev, mask);
66251a8f9d7SAlvaro Karsz }
66351a8f9d7SAlvaro Karsz
66451a8f9d7SAlvaro Karsz /* Read SNET config from PCI BAR */
psnet_read_cfg(struct pci_dev * pdev,struct psnet * psnet)66551a8f9d7SAlvaro Karsz static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet)
66651a8f9d7SAlvaro Karsz {
66751a8f9d7SAlvaro Karsz struct snet_cfg *cfg = &psnet->cfg;
66851a8f9d7SAlvaro Karsz u32 i, off;
66951a8f9d7SAlvaro Karsz int barno;
67051a8f9d7SAlvaro Karsz
67151a8f9d7SAlvaro Karsz /* Move to where the config starts */
67251a8f9d7SAlvaro Karsz off = SNET_CONFIG_OFF;
67351a8f9d7SAlvaro Karsz
67451a8f9d7SAlvaro Karsz /* Find BAR used for communication */
67551a8f9d7SAlvaro Karsz barno = psnet_detect_bar(psnet, off);
67651a8f9d7SAlvaro Karsz if (barno < 0) {
67751a8f9d7SAlvaro Karsz SNET_ERR(pdev, "SNET config is not ready.\n");
67851a8f9d7SAlvaro Karsz return barno;
67951a8f9d7SAlvaro Karsz }
68051a8f9d7SAlvaro Karsz
68151a8f9d7SAlvaro Karsz /* Save used BAR number and unmap all other BARs */
68251a8f9d7SAlvaro Karsz psnet->barno = barno;
68351a8f9d7SAlvaro Karsz SNET_DBG(pdev, "Using BAR number %d\n", barno);
68451a8f9d7SAlvaro Karsz
68551a8f9d7SAlvaro Karsz psnet_unmap_unused_bars(pdev, psnet);
68651a8f9d7SAlvaro Karsz
68751a8f9d7SAlvaro Karsz /* load config from BAR */
68851a8f9d7SAlvaro Karsz cfg->key = psnet_read32(psnet, off);
68951a8f9d7SAlvaro Karsz off += 4;
69051a8f9d7SAlvaro Karsz cfg->cfg_size = psnet_read32(psnet, off);
69151a8f9d7SAlvaro Karsz off += 4;
69251a8f9d7SAlvaro Karsz cfg->cfg_ver = psnet_read32(psnet, off);
69351a8f9d7SAlvaro Karsz off += 4;
69451a8f9d7SAlvaro Karsz /* The negotiated config version is the lower one between this driver's config
69551a8f9d7SAlvaro Karsz * and the DPU's.
69651a8f9d7SAlvaro Karsz */
69751a8f9d7SAlvaro Karsz psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION);
69851a8f9d7SAlvaro Karsz SNET_DBG(pdev, "SNET config version %u\n", psnet->negotiated_cfg_ver);
69951a8f9d7SAlvaro Karsz
70051a8f9d7SAlvaro Karsz cfg->vf_num = psnet_read32(psnet, off);
70151a8f9d7SAlvaro Karsz off += 4;
70251a8f9d7SAlvaro Karsz cfg->vf_bar = psnet_read32(psnet, off);
70351a8f9d7SAlvaro Karsz off += 4;
70451a8f9d7SAlvaro Karsz cfg->host_cfg_off = psnet_read32(psnet, off);
70551a8f9d7SAlvaro Karsz off += 4;
70651a8f9d7SAlvaro Karsz cfg->max_size_host_cfg = psnet_read32(psnet, off);
70751a8f9d7SAlvaro Karsz off += 4;
70851a8f9d7SAlvaro Karsz cfg->virtio_cfg_off = psnet_read32(psnet, off);
70951a8f9d7SAlvaro Karsz off += 4;
71051a8f9d7SAlvaro Karsz cfg->kick_off = psnet_read32(psnet, off);
71151a8f9d7SAlvaro Karsz off += 4;
71251a8f9d7SAlvaro Karsz cfg->hwmon_off = psnet_read32(psnet, off);
71351a8f9d7SAlvaro Karsz off += 4;
7143f3a1675SAlvaro Karsz cfg->ctrl_off = psnet_read32(psnet, off);
71551a8f9d7SAlvaro Karsz off += 4;
71651a8f9d7SAlvaro Karsz cfg->flags = psnet_read32(psnet, off);
71751a8f9d7SAlvaro Karsz off += 4;
71851a8f9d7SAlvaro Karsz /* Ignore Reserved */
71951a8f9d7SAlvaro Karsz off += sizeof(cfg->rsvd);
72051a8f9d7SAlvaro Karsz
72151a8f9d7SAlvaro Karsz cfg->devices_num = psnet_read32(psnet, off);
72251a8f9d7SAlvaro Karsz off += 4;
72351a8f9d7SAlvaro Karsz /* Allocate memory to hold pointer to the devices */
72451a8f9d7SAlvaro Karsz cfg->devs = kcalloc(cfg->devices_num, sizeof(void *), GFP_KERNEL);
72551a8f9d7SAlvaro Karsz if (!cfg->devs)
72651a8f9d7SAlvaro Karsz return -ENOMEM;
72751a8f9d7SAlvaro Karsz
72851a8f9d7SAlvaro Karsz /* Load device configuration from BAR */
72951a8f9d7SAlvaro Karsz for (i = 0; i < cfg->devices_num; i++) {
73051a8f9d7SAlvaro Karsz cfg->devs[i] = kzalloc(sizeof(*cfg->devs[i]), GFP_KERNEL);
73151a8f9d7SAlvaro Karsz if (!cfg->devs[i]) {
73251a8f9d7SAlvaro Karsz snet_free_cfg(cfg);
73351a8f9d7SAlvaro Karsz return -ENOMEM;
73451a8f9d7SAlvaro Karsz }
73551a8f9d7SAlvaro Karsz /* Read device config */
73651a8f9d7SAlvaro Karsz cfg->devs[i]->virtio_id = psnet_read32(psnet, off);
73751a8f9d7SAlvaro Karsz off += 4;
73851a8f9d7SAlvaro Karsz cfg->devs[i]->vq_num = psnet_read32(psnet, off);
73951a8f9d7SAlvaro Karsz off += 4;
74051a8f9d7SAlvaro Karsz cfg->devs[i]->vq_size = psnet_read32(psnet, off);
74151a8f9d7SAlvaro Karsz off += 4;
74251a8f9d7SAlvaro Karsz cfg->devs[i]->vfid = psnet_read32(psnet, off);
74351a8f9d7SAlvaro Karsz off += 4;
74451a8f9d7SAlvaro Karsz cfg->devs[i]->features = psnet_read64(psnet, off);
74551a8f9d7SAlvaro Karsz off += 8;
74651a8f9d7SAlvaro Karsz /* Ignore Reserved */
74751a8f9d7SAlvaro Karsz off += sizeof(cfg->devs[i]->rsvd);
74851a8f9d7SAlvaro Karsz
74951a8f9d7SAlvaro Karsz cfg->devs[i]->cfg_size = psnet_read32(psnet, off);
75051a8f9d7SAlvaro Karsz off += 4;
75151a8f9d7SAlvaro Karsz
75251a8f9d7SAlvaro Karsz /* Is the config witten to the DPU going to be too big? */
75351a8f9d7SAlvaro Karsz if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num >
75451a8f9d7SAlvaro Karsz cfg->max_size_host_cfg) {
75551a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to read SNET config, the config is too big..\n");
75651a8f9d7SAlvaro Karsz snet_free_cfg(cfg);
75751a8f9d7SAlvaro Karsz return -EINVAL;
75851a8f9d7SAlvaro Karsz }
75951a8f9d7SAlvaro Karsz }
76051a8f9d7SAlvaro Karsz return 0;
76151a8f9d7SAlvaro Karsz }
76251a8f9d7SAlvaro Karsz
psnet_alloc_irq_vector(struct pci_dev * pdev,struct psnet * psnet)76351a8f9d7SAlvaro Karsz static int psnet_alloc_irq_vector(struct pci_dev *pdev, struct psnet *psnet)
76451a8f9d7SAlvaro Karsz {
76551a8f9d7SAlvaro Karsz int ret = 0;
76651a8f9d7SAlvaro Karsz u32 i, irq_num = 0;
76751a8f9d7SAlvaro Karsz
76851a8f9d7SAlvaro Karsz /* Let's count how many IRQs we need, 1 for every VQ + 1 for config change */
76951a8f9d7SAlvaro Karsz for (i = 0; i < psnet->cfg.devices_num; i++)
77051a8f9d7SAlvaro Karsz irq_num += psnet->cfg.devs[i]->vq_num + 1;
77151a8f9d7SAlvaro Karsz
77251a8f9d7SAlvaro Karsz ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX);
77351a8f9d7SAlvaro Karsz if (ret != irq_num) {
77451a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to allocate IRQ vectors\n");
77551a8f9d7SAlvaro Karsz return ret;
77651a8f9d7SAlvaro Karsz }
77751a8f9d7SAlvaro Karsz SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n", irq_num);
77851a8f9d7SAlvaro Karsz
77951a8f9d7SAlvaro Karsz return 0;
78051a8f9d7SAlvaro Karsz }
78151a8f9d7SAlvaro Karsz
snet_alloc_irq_vector(struct pci_dev * pdev,struct snet_dev_cfg * snet_cfg)78251a8f9d7SAlvaro Karsz static int snet_alloc_irq_vector(struct pci_dev *pdev, struct snet_dev_cfg *snet_cfg)
78351a8f9d7SAlvaro Karsz {
78451a8f9d7SAlvaro Karsz int ret = 0;
78551a8f9d7SAlvaro Karsz u32 irq_num;
78651a8f9d7SAlvaro Karsz
78751a8f9d7SAlvaro Karsz /* We want 1 IRQ for every VQ + 1 for config change events */
78851a8f9d7SAlvaro Karsz irq_num = snet_cfg->vq_num + 1;
78951a8f9d7SAlvaro Karsz
79051a8f9d7SAlvaro Karsz ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX);
79151a8f9d7SAlvaro Karsz if (ret <= 0) {
79251a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to allocate IRQ vectors\n");
79351a8f9d7SAlvaro Karsz return ret;
79451a8f9d7SAlvaro Karsz }
79551a8f9d7SAlvaro Karsz
79651a8f9d7SAlvaro Karsz return 0;
79751a8f9d7SAlvaro Karsz }
79851a8f9d7SAlvaro Karsz
snet_free_vqs(struct snet * snet)79951a8f9d7SAlvaro Karsz static void snet_free_vqs(struct snet *snet)
80051a8f9d7SAlvaro Karsz {
80151a8f9d7SAlvaro Karsz u32 i;
80251a8f9d7SAlvaro Karsz
80351a8f9d7SAlvaro Karsz if (!snet->vqs)
80451a8f9d7SAlvaro Karsz return;
80551a8f9d7SAlvaro Karsz
80651a8f9d7SAlvaro Karsz for (i = 0 ; i < snet->cfg->vq_num ; i++) {
80751a8f9d7SAlvaro Karsz if (!snet->vqs[i])
80851a8f9d7SAlvaro Karsz break;
80951a8f9d7SAlvaro Karsz
81051a8f9d7SAlvaro Karsz kfree(snet->vqs[i]);
81151a8f9d7SAlvaro Karsz }
81251a8f9d7SAlvaro Karsz kfree(snet->vqs);
81351a8f9d7SAlvaro Karsz }
81451a8f9d7SAlvaro Karsz
snet_build_vqs(struct snet * snet)81551a8f9d7SAlvaro Karsz static int snet_build_vqs(struct snet *snet)
81651a8f9d7SAlvaro Karsz {
81751a8f9d7SAlvaro Karsz u32 i;
81851a8f9d7SAlvaro Karsz /* Allocate the VQ pointers array */
81951a8f9d7SAlvaro Karsz snet->vqs = kcalloc(snet->cfg->vq_num, sizeof(void *), GFP_KERNEL);
82051a8f9d7SAlvaro Karsz if (!snet->vqs)
82151a8f9d7SAlvaro Karsz return -ENOMEM;
82251a8f9d7SAlvaro Karsz
82351a8f9d7SAlvaro Karsz /* Allocate the VQs */
82451a8f9d7SAlvaro Karsz for (i = 0; i < snet->cfg->vq_num; i++) {
82551a8f9d7SAlvaro Karsz snet->vqs[i] = kzalloc(sizeof(*snet->vqs[i]), GFP_KERNEL);
82651a8f9d7SAlvaro Karsz if (!snet->vqs[i]) {
82751a8f9d7SAlvaro Karsz snet_free_vqs(snet);
82851a8f9d7SAlvaro Karsz return -ENOMEM;
82951a8f9d7SAlvaro Karsz }
83051a8f9d7SAlvaro Karsz /* Reset IRQ num */
83151a8f9d7SAlvaro Karsz snet->vqs[i]->irq = -1;
83251a8f9d7SAlvaro Karsz /* VQ serial ID */
83351a8f9d7SAlvaro Karsz snet->vqs[i]->sid = i;
83451a8f9d7SAlvaro Karsz /* Kick address - every VQ gets 4B */
83551a8f9d7SAlvaro Karsz snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off +
83651a8f9d7SAlvaro Karsz snet->vqs[i]->sid * 4;
83751a8f9d7SAlvaro Karsz /* Clear kick address for this VQ */
83851a8f9d7SAlvaro Karsz iowrite32(0, snet->vqs[i]->kick_ptr);
83951a8f9d7SAlvaro Karsz }
84051a8f9d7SAlvaro Karsz return 0;
84151a8f9d7SAlvaro Karsz }
84251a8f9d7SAlvaro Karsz
psnet_get_next_irq_num(struct psnet * psnet)84351a8f9d7SAlvaro Karsz static int psnet_get_next_irq_num(struct psnet *psnet)
84451a8f9d7SAlvaro Karsz {
84551a8f9d7SAlvaro Karsz int irq;
84651a8f9d7SAlvaro Karsz
84751a8f9d7SAlvaro Karsz spin_lock(&psnet->lock);
84851a8f9d7SAlvaro Karsz irq = psnet->next_irq++;
84951a8f9d7SAlvaro Karsz spin_unlock(&psnet->lock);
85051a8f9d7SAlvaro Karsz
85151a8f9d7SAlvaro Karsz return irq;
85251a8f9d7SAlvaro Karsz }
85351a8f9d7SAlvaro Karsz
snet_reserve_irq_idx(struct pci_dev * pdev,struct snet * snet)85451a8f9d7SAlvaro Karsz static void snet_reserve_irq_idx(struct pci_dev *pdev, struct snet *snet)
85551a8f9d7SAlvaro Karsz {
85651a8f9d7SAlvaro Karsz struct psnet *psnet = snet->psnet;
85751a8f9d7SAlvaro Karsz int i;
85851a8f9d7SAlvaro Karsz
85951a8f9d7SAlvaro Karsz /* one IRQ for every VQ, and one for config changes */
86051a8f9d7SAlvaro Karsz snet->cfg_irq_idx = psnet_get_next_irq_num(psnet);
86151a8f9d7SAlvaro Karsz snprintf(snet->cfg_irq_name, SNET_NAME_SIZE, "snet[%s]-cfg[%d]",
86251a8f9d7SAlvaro Karsz pci_name(pdev), snet->cfg_irq_idx);
86351a8f9d7SAlvaro Karsz
86451a8f9d7SAlvaro Karsz for (i = 0; i < snet->cfg->vq_num; i++) {
86551a8f9d7SAlvaro Karsz /* Get next free IRQ ID */
86651a8f9d7SAlvaro Karsz snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet);
86751a8f9d7SAlvaro Karsz /* Write IRQ name */
86851a8f9d7SAlvaro Karsz snprintf(snet->vqs[i]->irq_name, SNET_NAME_SIZE, "snet[%s]-vq[%d]",
86951a8f9d7SAlvaro Karsz pci_name(pdev), snet->vqs[i]->irq_idx);
87051a8f9d7SAlvaro Karsz }
87151a8f9d7SAlvaro Karsz }
87251a8f9d7SAlvaro Karsz
87351a8f9d7SAlvaro Karsz /* Find a device config based on virtual function id */
snet_find_dev_cfg(struct snet_cfg * cfg,u32 vfid)87451a8f9d7SAlvaro Karsz static struct snet_dev_cfg *snet_find_dev_cfg(struct snet_cfg *cfg, u32 vfid)
87551a8f9d7SAlvaro Karsz {
87651a8f9d7SAlvaro Karsz u32 i;
87751a8f9d7SAlvaro Karsz
87851a8f9d7SAlvaro Karsz for (i = 0; i < cfg->devices_num; i++) {
87951a8f9d7SAlvaro Karsz if (cfg->devs[i]->vfid == vfid)
88051a8f9d7SAlvaro Karsz return cfg->devs[i];
88151a8f9d7SAlvaro Karsz }
88251a8f9d7SAlvaro Karsz /* Oppss.. no config found.. */
88351a8f9d7SAlvaro Karsz return NULL;
88451a8f9d7SAlvaro Karsz }
88551a8f9d7SAlvaro Karsz
88651a8f9d7SAlvaro Karsz /* Probe function for a physical PCI function */
snet_vdpa_probe_pf(struct pci_dev * pdev)88751a8f9d7SAlvaro Karsz static int snet_vdpa_probe_pf(struct pci_dev *pdev)
88851a8f9d7SAlvaro Karsz {
88951a8f9d7SAlvaro Karsz struct psnet *psnet;
89051a8f9d7SAlvaro Karsz int ret = 0;
89151a8f9d7SAlvaro Karsz bool pf_irqs = false;
89251a8f9d7SAlvaro Karsz
89351a8f9d7SAlvaro Karsz ret = pcim_enable_device(pdev);
89451a8f9d7SAlvaro Karsz if (ret) {
89551a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to enable PCI device\n");
89651a8f9d7SAlvaro Karsz return ret;
89751a8f9d7SAlvaro Karsz }
89851a8f9d7SAlvaro Karsz
89951a8f9d7SAlvaro Karsz /* Allocate a PCI physical function device */
90051a8f9d7SAlvaro Karsz psnet = kzalloc(sizeof(*psnet), GFP_KERNEL);
90151a8f9d7SAlvaro Karsz if (!psnet)
90251a8f9d7SAlvaro Karsz return -ENOMEM;
90351a8f9d7SAlvaro Karsz
90451a8f9d7SAlvaro Karsz /* Init PSNET spinlock */
90551a8f9d7SAlvaro Karsz spin_lock_init(&psnet->lock);
90651a8f9d7SAlvaro Karsz
90751a8f9d7SAlvaro Karsz pci_set_master(pdev);
90851a8f9d7SAlvaro Karsz pci_set_drvdata(pdev, psnet);
90951a8f9d7SAlvaro Karsz
91051a8f9d7SAlvaro Karsz /* Open SNET MAIN BAR */
91151a8f9d7SAlvaro Karsz ret = psnet_open_pf_bar(pdev, psnet);
91251a8f9d7SAlvaro Karsz if (ret)
91351a8f9d7SAlvaro Karsz goto free_psnet;
91451a8f9d7SAlvaro Karsz
91551a8f9d7SAlvaro Karsz /* Try to read SNET's config from PCI BAR */
91651a8f9d7SAlvaro Karsz ret = psnet_read_cfg(pdev, psnet);
91751a8f9d7SAlvaro Karsz if (ret)
91851a8f9d7SAlvaro Karsz goto free_psnet;
91951a8f9d7SAlvaro Karsz
92051a8f9d7SAlvaro Karsz /* If SNET_CFG_FLAG_IRQ_PF flag is set, we should use
92151a8f9d7SAlvaro Karsz * PF MSI-X vectors
92251a8f9d7SAlvaro Karsz */
92351a8f9d7SAlvaro Karsz pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);
92451a8f9d7SAlvaro Karsz
92551a8f9d7SAlvaro Karsz if (pf_irqs) {
92651a8f9d7SAlvaro Karsz ret = psnet_alloc_irq_vector(pdev, psnet);
92751a8f9d7SAlvaro Karsz if (ret)
92851a8f9d7SAlvaro Karsz goto free_cfg;
92951a8f9d7SAlvaro Karsz }
93051a8f9d7SAlvaro Karsz
93151a8f9d7SAlvaro Karsz SNET_DBG(pdev, "Enable %u virtual functions\n", psnet->cfg.vf_num);
93251a8f9d7SAlvaro Karsz ret = pci_enable_sriov(pdev, psnet->cfg.vf_num);
93351a8f9d7SAlvaro Karsz if (ret) {
93451a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to enable SR-IOV\n");
93551a8f9d7SAlvaro Karsz goto free_irq;
93651a8f9d7SAlvaro Karsz }
93751a8f9d7SAlvaro Karsz
93851a8f9d7SAlvaro Karsz /* Create HW monitor device */
93951a8f9d7SAlvaro Karsz if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_HWMON)) {
94051a8f9d7SAlvaro Karsz #if IS_ENABLED(CONFIG_HWMON)
94151a8f9d7SAlvaro Karsz psnet_create_hwmon(pdev);
94251a8f9d7SAlvaro Karsz #else
94351a8f9d7SAlvaro Karsz SNET_WARN(pdev, "Can't start HWMON, CONFIG_HWMON is not enabled\n");
94451a8f9d7SAlvaro Karsz #endif
94551a8f9d7SAlvaro Karsz }
94651a8f9d7SAlvaro Karsz
94751a8f9d7SAlvaro Karsz return 0;
94851a8f9d7SAlvaro Karsz
94951a8f9d7SAlvaro Karsz free_irq:
95051a8f9d7SAlvaro Karsz if (pf_irqs)
95151a8f9d7SAlvaro Karsz pci_free_irq_vectors(pdev);
95251a8f9d7SAlvaro Karsz free_cfg:
95351a8f9d7SAlvaro Karsz snet_free_cfg(&psnet->cfg);
95451a8f9d7SAlvaro Karsz free_psnet:
95551a8f9d7SAlvaro Karsz kfree(psnet);
95651a8f9d7SAlvaro Karsz return ret;
95751a8f9d7SAlvaro Karsz }
95851a8f9d7SAlvaro Karsz
95951a8f9d7SAlvaro Karsz /* Probe function for a virtual PCI function */
snet_vdpa_probe_vf(struct pci_dev * pdev)96051a8f9d7SAlvaro Karsz static int snet_vdpa_probe_vf(struct pci_dev *pdev)
96151a8f9d7SAlvaro Karsz {
96251a8f9d7SAlvaro Karsz struct pci_dev *pdev_pf = pdev->physfn;
96351a8f9d7SAlvaro Karsz struct psnet *psnet = pci_get_drvdata(pdev_pf);
96451a8f9d7SAlvaro Karsz struct snet_dev_cfg *dev_cfg;
96551a8f9d7SAlvaro Karsz struct snet *snet;
96651a8f9d7SAlvaro Karsz u32 vfid;
96751a8f9d7SAlvaro Karsz int ret;
96851a8f9d7SAlvaro Karsz bool pf_irqs = false;
96951a8f9d7SAlvaro Karsz
97051a8f9d7SAlvaro Karsz /* Get virtual function id.
97151a8f9d7SAlvaro Karsz * (the DPU counts the VFs from 1)
97251a8f9d7SAlvaro Karsz */
97351a8f9d7SAlvaro Karsz ret = pci_iov_vf_id(pdev);
97451a8f9d7SAlvaro Karsz if (ret < 0) {
97551a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to find a VF id\n");
97651a8f9d7SAlvaro Karsz return ret;
97751a8f9d7SAlvaro Karsz }
97851a8f9d7SAlvaro Karsz vfid = ret + 1;
97951a8f9d7SAlvaro Karsz
98051a8f9d7SAlvaro Karsz /* Find the snet_dev_cfg based on vfid */
98151a8f9d7SAlvaro Karsz dev_cfg = snet_find_dev_cfg(&psnet->cfg, vfid);
98251a8f9d7SAlvaro Karsz if (!dev_cfg) {
98351a8f9d7SAlvaro Karsz SNET_WARN(pdev, "Failed to find a VF config..\n");
98451a8f9d7SAlvaro Karsz return -ENODEV;
98551a8f9d7SAlvaro Karsz }
98651a8f9d7SAlvaro Karsz
98751a8f9d7SAlvaro Karsz /* Which PCI device should allocate the IRQs?
98851a8f9d7SAlvaro Karsz * If the SNET_CFG_FLAG_IRQ_PF flag set, the PF device allocates the IRQs
98951a8f9d7SAlvaro Karsz */
99051a8f9d7SAlvaro Karsz pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);
99151a8f9d7SAlvaro Karsz
99251a8f9d7SAlvaro Karsz ret = pcim_enable_device(pdev);
99351a8f9d7SAlvaro Karsz if (ret) {
99451a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to enable PCI VF device\n");
99551a8f9d7SAlvaro Karsz return ret;
99651a8f9d7SAlvaro Karsz }
99751a8f9d7SAlvaro Karsz
99851a8f9d7SAlvaro Karsz /* Request for MSI-X IRQs */
99951a8f9d7SAlvaro Karsz if (!pf_irqs) {
100051a8f9d7SAlvaro Karsz ret = snet_alloc_irq_vector(pdev, dev_cfg);
100151a8f9d7SAlvaro Karsz if (ret)
100251a8f9d7SAlvaro Karsz return ret;
100351a8f9d7SAlvaro Karsz }
100451a8f9d7SAlvaro Karsz
100551a8f9d7SAlvaro Karsz /* Allocate vdpa device */
100651a8f9d7SAlvaro Karsz snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL,
100751a8f9d7SAlvaro Karsz false);
100851a8f9d7SAlvaro Karsz if (!snet) {
100951a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to allocate a vdpa device\n");
101051a8f9d7SAlvaro Karsz ret = -ENOMEM;
101151a8f9d7SAlvaro Karsz goto free_irqs;
101251a8f9d7SAlvaro Karsz }
101351a8f9d7SAlvaro Karsz
10143f3a1675SAlvaro Karsz /* Init control mutex and spinlock */
10153f3a1675SAlvaro Karsz mutex_init(&snet->ctrl_lock);
10163f3a1675SAlvaro Karsz spin_lock_init(&snet->ctrl_spinlock);
10173f3a1675SAlvaro Karsz
101851a8f9d7SAlvaro Karsz /* Save pci device pointer */
101951a8f9d7SAlvaro Karsz snet->pdev = pdev;
102051a8f9d7SAlvaro Karsz snet->psnet = psnet;
102151a8f9d7SAlvaro Karsz snet->cfg = dev_cfg;
102251a8f9d7SAlvaro Karsz snet->dpu_ready = false;
102351a8f9d7SAlvaro Karsz snet->sid = vfid;
102451a8f9d7SAlvaro Karsz /* Reset IRQ value */
102551a8f9d7SAlvaro Karsz snet->cfg_irq = -1;
102651a8f9d7SAlvaro Karsz
102751a8f9d7SAlvaro Karsz ret = snet_open_vf_bar(pdev, snet);
102851a8f9d7SAlvaro Karsz if (ret)
102951a8f9d7SAlvaro Karsz goto put_device;
103051a8f9d7SAlvaro Karsz
103151a8f9d7SAlvaro Karsz /* Create a VirtIO config pointer */
103251a8f9d7SAlvaro Karsz snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off;
103351a8f9d7SAlvaro Karsz
10343f3a1675SAlvaro Karsz /* Clear control registers */
10353f3a1675SAlvaro Karsz snet_ctrl_clear(snet);
10363f3a1675SAlvaro Karsz
103751a8f9d7SAlvaro Karsz pci_set_master(pdev);
103851a8f9d7SAlvaro Karsz pci_set_drvdata(pdev, snet);
103951a8f9d7SAlvaro Karsz
104051a8f9d7SAlvaro Karsz ret = snet_build_vqs(snet);
104151a8f9d7SAlvaro Karsz if (ret)
104251a8f9d7SAlvaro Karsz goto put_device;
104351a8f9d7SAlvaro Karsz
104451a8f9d7SAlvaro Karsz /* Reserve IRQ indexes,
104551a8f9d7SAlvaro Karsz * The IRQs may be requested and freed multiple times,
104651a8f9d7SAlvaro Karsz * but the indexes won't change.
104751a8f9d7SAlvaro Karsz */
104851a8f9d7SAlvaro Karsz snet_reserve_irq_idx(pf_irqs ? pdev_pf : pdev, snet);
104951a8f9d7SAlvaro Karsz
105051a8f9d7SAlvaro Karsz /*set DMA device*/
105151a8f9d7SAlvaro Karsz snet->vdpa.dma_dev = &pdev->dev;
105251a8f9d7SAlvaro Karsz
105351a8f9d7SAlvaro Karsz /* Register VDPA device */
105451a8f9d7SAlvaro Karsz ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num);
105551a8f9d7SAlvaro Karsz if (ret) {
105651a8f9d7SAlvaro Karsz SNET_ERR(pdev, "Failed to register vdpa device\n");
105751a8f9d7SAlvaro Karsz goto free_vqs;
105851a8f9d7SAlvaro Karsz }
105951a8f9d7SAlvaro Karsz
106051a8f9d7SAlvaro Karsz return 0;
106151a8f9d7SAlvaro Karsz
106251a8f9d7SAlvaro Karsz free_vqs:
106351a8f9d7SAlvaro Karsz snet_free_vqs(snet);
106451a8f9d7SAlvaro Karsz put_device:
106551a8f9d7SAlvaro Karsz put_device(&snet->vdpa.dev);
106651a8f9d7SAlvaro Karsz free_irqs:
106751a8f9d7SAlvaro Karsz if (!pf_irqs)
106851a8f9d7SAlvaro Karsz pci_free_irq_vectors(pdev);
106951a8f9d7SAlvaro Karsz return ret;
107051a8f9d7SAlvaro Karsz }
107151a8f9d7SAlvaro Karsz
snet_vdpa_probe(struct pci_dev * pdev,const struct pci_device_id * id)107251a8f9d7SAlvaro Karsz static int snet_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
107351a8f9d7SAlvaro Karsz {
107451a8f9d7SAlvaro Karsz if (pdev->is_virtfn)
107551a8f9d7SAlvaro Karsz return snet_vdpa_probe_vf(pdev);
107651a8f9d7SAlvaro Karsz else
107751a8f9d7SAlvaro Karsz return snet_vdpa_probe_pf(pdev);
107851a8f9d7SAlvaro Karsz }
107951a8f9d7SAlvaro Karsz
snet_vdpa_remove_pf(struct pci_dev * pdev)108051a8f9d7SAlvaro Karsz static void snet_vdpa_remove_pf(struct pci_dev *pdev)
108151a8f9d7SAlvaro Karsz {
108251a8f9d7SAlvaro Karsz struct psnet *psnet = pci_get_drvdata(pdev);
108351a8f9d7SAlvaro Karsz
108451a8f9d7SAlvaro Karsz pci_disable_sriov(pdev);
108551a8f9d7SAlvaro Karsz /* If IRQs are allocated from the PF, we should free the IRQs */
108651a8f9d7SAlvaro Karsz if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))
108751a8f9d7SAlvaro Karsz pci_free_irq_vectors(pdev);
108851a8f9d7SAlvaro Karsz
108951a8f9d7SAlvaro Karsz snet_free_cfg(&psnet->cfg);
109051a8f9d7SAlvaro Karsz kfree(psnet);
109151a8f9d7SAlvaro Karsz }
109251a8f9d7SAlvaro Karsz
snet_vdpa_remove_vf(struct pci_dev * pdev)109351a8f9d7SAlvaro Karsz static void snet_vdpa_remove_vf(struct pci_dev *pdev)
109451a8f9d7SAlvaro Karsz {
109551a8f9d7SAlvaro Karsz struct snet *snet = pci_get_drvdata(pdev);
109651a8f9d7SAlvaro Karsz struct psnet *psnet = snet->psnet;
109751a8f9d7SAlvaro Karsz
109851a8f9d7SAlvaro Karsz vdpa_unregister_device(&snet->vdpa);
109951a8f9d7SAlvaro Karsz snet_free_vqs(snet);
110051a8f9d7SAlvaro Karsz /* If IRQs are allocated from the VF, we should free the IRQs */
110151a8f9d7SAlvaro Karsz if (!PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))
110251a8f9d7SAlvaro Karsz pci_free_irq_vectors(pdev);
110351a8f9d7SAlvaro Karsz }
110451a8f9d7SAlvaro Karsz
snet_vdpa_remove(struct pci_dev * pdev)110551a8f9d7SAlvaro Karsz static void snet_vdpa_remove(struct pci_dev *pdev)
110651a8f9d7SAlvaro Karsz {
110751a8f9d7SAlvaro Karsz if (pdev->is_virtfn)
110851a8f9d7SAlvaro Karsz snet_vdpa_remove_vf(pdev);
110951a8f9d7SAlvaro Karsz else
111051a8f9d7SAlvaro Karsz snet_vdpa_remove_pf(pdev);
111151a8f9d7SAlvaro Karsz }
111251a8f9d7SAlvaro Karsz
111351a8f9d7SAlvaro Karsz static struct pci_device_id snet_driver_pci_ids[] = {
111451a8f9d7SAlvaro Karsz { PCI_DEVICE_SUB(PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID,
111551a8f9d7SAlvaro Karsz PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID) },
111651a8f9d7SAlvaro Karsz { 0 },
111751a8f9d7SAlvaro Karsz };
111851a8f9d7SAlvaro Karsz
111951a8f9d7SAlvaro Karsz MODULE_DEVICE_TABLE(pci, snet_driver_pci_ids);
112051a8f9d7SAlvaro Karsz
112151a8f9d7SAlvaro Karsz static struct pci_driver snet_vdpa_driver = {
112251a8f9d7SAlvaro Karsz .name = "snet-vdpa-driver",
112351a8f9d7SAlvaro Karsz .id_table = snet_driver_pci_ids,
112451a8f9d7SAlvaro Karsz .probe = snet_vdpa_probe,
112551a8f9d7SAlvaro Karsz .remove = snet_vdpa_remove,
112651a8f9d7SAlvaro Karsz };
112751a8f9d7SAlvaro Karsz
112851a8f9d7SAlvaro Karsz module_pci_driver(snet_vdpa_driver);
112951a8f9d7SAlvaro Karsz
113051a8f9d7SAlvaro Karsz MODULE_AUTHOR("Alvaro Karsz <alvaro.karsz@solid-run.com>");
113151a8f9d7SAlvaro Karsz MODULE_DESCRIPTION("SolidRun vDPA driver");
113251a8f9d7SAlvaro Karsz MODULE_LICENSE("GPL v2");
1133