11a86b377SEli Cohen // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 21a86b377SEli Cohen /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 31a86b377SEli Cohen 41a86b377SEli Cohen #include <linux/vdpa.h> 51a86b377SEli Cohen #include <uapi/linux/virtio_ids.h> 61a86b377SEli Cohen #include <linux/virtio_config.h> 71a86b377SEli Cohen #include <linux/mlx5/qp.h> 81a86b377SEli Cohen #include <linux/mlx5/device.h> 91a86b377SEli Cohen #include <linux/mlx5/vport.h> 101a86b377SEli Cohen #include <linux/mlx5/fs.h> 111a86b377SEli Cohen #include <linux/mlx5/device.h> 121a86b377SEli Cohen #include "mlx5_vnet.h" 131a86b377SEli Cohen #include "mlx5_vdpa_ifc.h" 141a86b377SEli Cohen #include "mlx5_vdpa.h" 151a86b377SEli Cohen 161a86b377SEli Cohen #define to_mvdev(__vdev) container_of((__vdev), struct mlx5_vdpa_dev, vdev) 171a86b377SEli Cohen 181a86b377SEli Cohen #define VALID_FEATURES_MASK \ 191a86b377SEli Cohen (BIT(VIRTIO_NET_F_CSUM) | BIT(VIRTIO_NET_F_GUEST_CSUM) | \ 201a86b377SEli Cohen BIT(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT(VIRTIO_NET_F_MTU) | BIT(VIRTIO_NET_F_MAC) | \ 211a86b377SEli Cohen BIT(VIRTIO_NET_F_GUEST_TSO4) | BIT(VIRTIO_NET_F_GUEST_TSO6) | \ 221a86b377SEli Cohen BIT(VIRTIO_NET_F_GUEST_ECN) | BIT(VIRTIO_NET_F_GUEST_UFO) | BIT(VIRTIO_NET_F_HOST_TSO4) | \ 231a86b377SEli Cohen BIT(VIRTIO_NET_F_HOST_TSO6) | BIT(VIRTIO_NET_F_HOST_ECN) | BIT(VIRTIO_NET_F_HOST_UFO) | \ 241a86b377SEli Cohen BIT(VIRTIO_NET_F_MRG_RXBUF) | BIT(VIRTIO_NET_F_STATUS) | BIT(VIRTIO_NET_F_CTRL_VQ) | \ 251a86b377SEli Cohen BIT(VIRTIO_NET_F_CTRL_RX) | BIT(VIRTIO_NET_F_CTRL_VLAN) | \ 261a86b377SEli Cohen BIT(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT(VIRTIO_NET_F_GUEST_ANNOUNCE) | \ 271a86b377SEli Cohen BIT(VIRTIO_NET_F_MQ) | BIT(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT(VIRTIO_NET_F_HASH_REPORT) | \ 281a86b377SEli Cohen BIT(VIRTIO_NET_F_RSS) | BIT(VIRTIO_NET_F_RSC_EXT) | BIT(VIRTIO_NET_F_STANDBY) | \ 291a86b377SEli Cohen BIT(VIRTIO_NET_F_SPEED_DUPLEX) | BIT(VIRTIO_F_NOTIFY_ON_EMPTY) | \ 301a86b377SEli Cohen BIT(VIRTIO_F_ANY_LAYOUT) | BIT(VIRTIO_F_VERSION_1) | BIT(VIRTIO_F_ACCESS_PLATFORM) | \ 311a86b377SEli Cohen BIT(VIRTIO_F_RING_PACKED) | BIT(VIRTIO_F_ORDER_PLATFORM) | BIT(VIRTIO_F_SR_IOV)) 321a86b377SEli Cohen 331a86b377SEli Cohen #define VALID_STATUS_MASK \ 341a86b377SEli Cohen (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \ 351a86b377SEli Cohen VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED) 361a86b377SEli Cohen 371a86b377SEli Cohen struct mlx5_vdpa_net_resources { 381a86b377SEli Cohen u32 tisn; 391a86b377SEli Cohen u32 tdn; 401a86b377SEli Cohen u32 tirn; 411a86b377SEli Cohen u32 rqtn; 421a86b377SEli Cohen bool valid; 431a86b377SEli Cohen }; 441a86b377SEli Cohen 451a86b377SEli Cohen struct mlx5_vdpa_cq_buf { 461a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 471a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 481a86b377SEli Cohen int cqe_size; 491a86b377SEli Cohen int nent; 501a86b377SEli Cohen }; 511a86b377SEli Cohen 521a86b377SEli Cohen struct mlx5_vdpa_cq { 531a86b377SEli Cohen struct mlx5_core_cq mcq; 541a86b377SEli Cohen struct mlx5_vdpa_cq_buf buf; 551a86b377SEli Cohen struct mlx5_db db; 561a86b377SEli Cohen int cqe; 571a86b377SEli Cohen }; 581a86b377SEli Cohen 591a86b377SEli Cohen struct mlx5_vdpa_umem { 601a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 611a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 621a86b377SEli Cohen int size; 631a86b377SEli Cohen u32 id; 641a86b377SEli Cohen }; 651a86b377SEli Cohen 661a86b377SEli Cohen struct mlx5_vdpa_qp { 671a86b377SEli Cohen struct mlx5_core_qp mqp; 681a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 691a86b377SEli Cohen struct mlx5_db db; 701a86b377SEli Cohen u16 head; 711a86b377SEli Cohen bool fw; 721a86b377SEli Cohen }; 731a86b377SEli Cohen 741a86b377SEli Cohen struct mlx5_vq_restore_info { 751a86b377SEli Cohen u32 num_ent; 761a86b377SEli Cohen u64 desc_addr; 771a86b377SEli Cohen u64 device_addr; 781a86b377SEli Cohen u64 driver_addr; 791a86b377SEli Cohen u16 avail_index; 801a86b377SEli Cohen bool ready; 811a86b377SEli Cohen struct vdpa_callback cb; 821a86b377SEli Cohen bool restore; 831a86b377SEli Cohen }; 841a86b377SEli Cohen 851a86b377SEli Cohen struct mlx5_vdpa_virtqueue { 861a86b377SEli Cohen bool ready; 871a86b377SEli Cohen u64 desc_addr; 881a86b377SEli Cohen u64 device_addr; 891a86b377SEli Cohen u64 driver_addr; 901a86b377SEli Cohen u32 num_ent; 911a86b377SEli Cohen struct vdpa_callback event_cb; 921a86b377SEli Cohen 931a86b377SEli Cohen /* Resources for implementing the notification channel from the device 941a86b377SEli Cohen * to the driver. fwqp is the firmware end of an RC connection; the 951a86b377SEli Cohen * other end is vqqp used by the driver. cq is is where completions are 961a86b377SEli Cohen * reported. 971a86b377SEli Cohen */ 981a86b377SEli Cohen struct mlx5_vdpa_cq cq; 991a86b377SEli Cohen struct mlx5_vdpa_qp fwqp; 1001a86b377SEli Cohen struct mlx5_vdpa_qp vqqp; 1011a86b377SEli Cohen 1021a86b377SEli Cohen /* umem resources are required for the virtqueue operation. They're use 1031a86b377SEli Cohen * is internal and they must be provided by the driver. 1041a86b377SEli Cohen */ 1051a86b377SEli Cohen struct mlx5_vdpa_umem umem1; 1061a86b377SEli Cohen struct mlx5_vdpa_umem umem2; 1071a86b377SEli Cohen struct mlx5_vdpa_umem umem3; 1081a86b377SEli Cohen 1091a86b377SEli Cohen bool initialized; 1101a86b377SEli Cohen int index; 1111a86b377SEli Cohen u32 virtq_id; 1121a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 1131a86b377SEli Cohen u16 avail_idx; 1141a86b377SEli Cohen int fw_state; 1151a86b377SEli Cohen 1161a86b377SEli Cohen /* keep last in the struct */ 1171a86b377SEli Cohen struct mlx5_vq_restore_info ri; 1181a86b377SEli Cohen }; 1191a86b377SEli Cohen 1201a86b377SEli Cohen /* We will remove this limitation once mlx5_vdpa_alloc_resources() 1211a86b377SEli Cohen * provides for driver space allocation 1221a86b377SEli Cohen */ 1231a86b377SEli Cohen #define MLX5_MAX_SUPPORTED_VQS 16 1241a86b377SEli Cohen 1251a86b377SEli Cohen struct mlx5_vdpa_net { 1261a86b377SEli Cohen struct mlx5_vdpa_dev mvdev; 1271a86b377SEli Cohen struct mlx5_vdpa_net_resources res; 1281a86b377SEli Cohen struct virtio_net_config config; 1291a86b377SEli Cohen struct mlx5_vdpa_virtqueue vqs[MLX5_MAX_SUPPORTED_VQS]; 1301a86b377SEli Cohen 1311a86b377SEli Cohen /* Serialize vq resources creation and destruction. This is required 1321a86b377SEli Cohen * since memory map might change and we need to destroy and create 1331a86b377SEli Cohen * resources while driver in operational. 1341a86b377SEli Cohen */ 1351a86b377SEli Cohen struct mutex reslock; 1361a86b377SEli Cohen struct mlx5_flow_table *rxft; 1371a86b377SEli Cohen struct mlx5_fc *rx_counter; 1381a86b377SEli Cohen struct mlx5_flow_handle *rx_rule; 1391a86b377SEli Cohen bool setup; 1401a86b377SEli Cohen }; 1411a86b377SEli Cohen 1421a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev); 1431a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev); 1441a86b377SEli Cohen static int setup_driver(struct mlx5_vdpa_net *ndev); 1451a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev); 1461a86b377SEli Cohen 1471a86b377SEli Cohen static bool mlx5_vdpa_debug; 1481a86b377SEli Cohen 1491a86b377SEli Cohen #define MLX5_LOG_VIO_FLAG(_feature) \ 1501a86b377SEli Cohen do { \ 1511a86b377SEli Cohen if (features & BIT(_feature)) \ 1521a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_feature); \ 1531a86b377SEli Cohen } while (0) 1541a86b377SEli Cohen 1551a86b377SEli Cohen #define MLX5_LOG_VIO_STAT(_status) \ 1561a86b377SEli Cohen do { \ 1571a86b377SEli Cohen if (status & (_status)) \ 1581a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_status); \ 1591a86b377SEli Cohen } while (0) 1601a86b377SEli Cohen 1611a86b377SEli Cohen static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set) 1621a86b377SEli Cohen { 1631a86b377SEli Cohen if (status & ~VALID_STATUS_MASK) 1641a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n", 1651a86b377SEli Cohen status & ~VALID_STATUS_MASK); 1661a86b377SEli Cohen 1671a86b377SEli Cohen if (!mlx5_vdpa_debug) 1681a86b377SEli Cohen return; 1691a86b377SEli Cohen 1701a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get"); 1711a86b377SEli Cohen if (set && !status) { 1721a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver resets the device\n"); 1731a86b377SEli Cohen return; 1741a86b377SEli Cohen } 1751a86b377SEli Cohen 1761a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE); 1771a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER); 1781a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK); 1791a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK); 1801a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET); 1811a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED); 1821a86b377SEli Cohen } 1831a86b377SEli Cohen 1841a86b377SEli Cohen static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set) 1851a86b377SEli Cohen { 1861a86b377SEli Cohen if (features & ~VALID_FEATURES_MASK) 1871a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n", 1881a86b377SEli Cohen features & ~VALID_FEATURES_MASK); 1891a86b377SEli Cohen 1901a86b377SEli Cohen if (!mlx5_vdpa_debug) 1911a86b377SEli Cohen return; 1921a86b377SEli Cohen 1931a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads"); 1941a86b377SEli Cohen if (!features) 1951a86b377SEli Cohen mlx5_vdpa_info(mvdev, "all feature bits are cleared\n"); 1961a86b377SEli Cohen 1971a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM); 1981a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM); 1991a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 2001a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU); 2011a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC); 2021a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4); 2031a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6); 2041a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN); 2051a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO); 2061a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4); 2071a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6); 2081a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN); 2091a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO); 2101a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF); 2111a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS); 2121a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ); 2131a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX); 2141a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN); 2151a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA); 2161a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE); 2171a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ); 2181a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR); 2191a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT); 2201a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS); 2211a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT); 2221a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY); 2231a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX); 2241a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY); 2251a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT); 2261a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1); 2271a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM); 2281a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED); 2291a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM); 2301a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV); 2311a86b377SEli Cohen } 2321a86b377SEli Cohen 2331a86b377SEli Cohen static int create_tis(struct mlx5_vdpa_net *ndev) 2341a86b377SEli Cohen { 2351a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; 2361a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 2371a86b377SEli Cohen void *tisc; 2381a86b377SEli Cohen int err; 2391a86b377SEli Cohen 2401a86b377SEli Cohen tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 2411a86b377SEli Cohen MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); 2421a86b377SEli Cohen err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); 2431a86b377SEli Cohen if (err) 2441a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err); 2451a86b377SEli Cohen 2461a86b377SEli Cohen return err; 2471a86b377SEli Cohen } 2481a86b377SEli Cohen 2491a86b377SEli Cohen static void destroy_tis(struct mlx5_vdpa_net *ndev) 2501a86b377SEli Cohen { 2511a86b377SEli Cohen mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); 2521a86b377SEli Cohen } 2531a86b377SEli Cohen 2541a86b377SEli Cohen #define MLX5_VDPA_CQE_SIZE 64 2551a86b377SEli Cohen #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE) 2561a86b377SEli Cohen 2571a86b377SEli Cohen static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) 2581a86b377SEli Cohen { 2591a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &buf->frag_buf; 2601a86b377SEli Cohen u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE; 2611a86b377SEli Cohen u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE; 2621a86b377SEli Cohen int err; 2631a86b377SEli Cohen 2641a86b377SEli Cohen err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, 2651a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 2661a86b377SEli Cohen if (err) 2671a86b377SEli Cohen return err; 2681a86b377SEli Cohen 2691a86b377SEli Cohen mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc); 2701a86b377SEli Cohen 2711a86b377SEli Cohen buf->cqe_size = MLX5_VDPA_CQE_SIZE; 2721a86b377SEli Cohen buf->nent = nent; 2731a86b377SEli Cohen 2741a86b377SEli Cohen return 0; 2751a86b377SEli Cohen } 2761a86b377SEli Cohen 2771a86b377SEli Cohen static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) 2781a86b377SEli Cohen { 2791a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &umem->frag_buf; 2801a86b377SEli Cohen 2811a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, 2821a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 2831a86b377SEli Cohen } 2841a86b377SEli Cohen 2851a86b377SEli Cohen static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) 2861a86b377SEli Cohen { 2871a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); 2881a86b377SEli Cohen } 2891a86b377SEli Cohen 2901a86b377SEli Cohen static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n) 2911a86b377SEli Cohen { 2921a86b377SEli Cohen return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n); 2931a86b377SEli Cohen } 2941a86b377SEli Cohen 2951a86b377SEli Cohen static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf) 2961a86b377SEli Cohen { 2971a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 2981a86b377SEli Cohen void *cqe; 2991a86b377SEli Cohen int i; 3001a86b377SEli Cohen 3011a86b377SEli Cohen for (i = 0; i < buf->nent; i++) { 3021a86b377SEli Cohen cqe = get_cqe(vcq, i); 3031a86b377SEli Cohen cqe64 = cqe; 3041a86b377SEli Cohen cqe64->op_own = MLX5_CQE_INVALID << 4; 3051a86b377SEli Cohen } 3061a86b377SEli Cohen } 3071a86b377SEli Cohen 3081a86b377SEli Cohen static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n) 3091a86b377SEli Cohen { 3101a86b377SEli Cohen struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1)); 3111a86b377SEli Cohen 3121a86b377SEli Cohen if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && 3131a86b377SEli Cohen !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe))) 3141a86b377SEli Cohen return cqe64; 3151a86b377SEli Cohen 3161a86b377SEli Cohen return NULL; 3171a86b377SEli Cohen } 3181a86b377SEli Cohen 3191a86b377SEli Cohen static void rx_post(struct mlx5_vdpa_qp *vqp, int n) 3201a86b377SEli Cohen { 3211a86b377SEli Cohen vqp->head += n; 3221a86b377SEli Cohen vqp->db.db[0] = cpu_to_be32(vqp->head); 3231a86b377SEli Cohen } 3241a86b377SEli Cohen 3251a86b377SEli Cohen static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, 3261a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq, u32 num_ent) 3271a86b377SEli Cohen { 3281a86b377SEli Cohen struct mlx5_vdpa_qp *vqp; 3291a86b377SEli Cohen __be64 *pas; 3301a86b377SEli Cohen void *qpc; 3311a86b377SEli Cohen 3321a86b377SEli Cohen vqp = fw ? &mvq->fwqp : &mvq->vqqp; 3331a86b377SEli Cohen MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); 3341a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 3351a86b377SEli Cohen if (vqp->fw) { 3361a86b377SEli Cohen /* Firmware QP is allocated by the driver for the firmware's 3371a86b377SEli Cohen * use so we can skip part of the params as they will be chosen by firmware 3381a86b377SEli Cohen */ 3391a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 3401a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ); 3411a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 3421a86b377SEli Cohen return; 3431a86b377SEli Cohen } 3441a86b377SEli Cohen 3451a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 3461a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 3471a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 3481a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 3491a86b377SEli Cohen MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); 3501a86b377SEli Cohen MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 3511a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 3521a86b377SEli Cohen MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn); 3531a86b377SEli Cohen MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent)); 3541a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 3551a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas); 3561a86b377SEli Cohen mlx5_fill_page_frag_array(&vqp->frag_buf, pas); 3571a86b377SEli Cohen } 3581a86b377SEli Cohen 3591a86b377SEli Cohen static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) 3601a86b377SEli Cohen { 3611a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, 3621a86b377SEli Cohen num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf, 3631a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3641a86b377SEli Cohen } 3651a86b377SEli Cohen 3661a86b377SEli Cohen static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 3671a86b377SEli Cohen { 3681a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); 3691a86b377SEli Cohen } 3701a86b377SEli Cohen 3711a86b377SEli Cohen static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 3721a86b377SEli Cohen struct mlx5_vdpa_qp *vqp) 3731a86b377SEli Cohen { 3741a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 3751a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_qp_in); 3761a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 3771a86b377SEli Cohen void *qpc; 3781a86b377SEli Cohen void *in; 3791a86b377SEli Cohen int err; 3801a86b377SEli Cohen 3811a86b377SEli Cohen if (!vqp->fw) { 3821a86b377SEli Cohen vqp = &mvq->vqqp; 3831a86b377SEli Cohen err = rq_buf_alloc(ndev, vqp, mvq->num_ent); 3841a86b377SEli Cohen if (err) 3851a86b377SEli Cohen return err; 3861a86b377SEli Cohen 3871a86b377SEli Cohen err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); 3881a86b377SEli Cohen if (err) 3891a86b377SEli Cohen goto err_db; 3901a86b377SEli Cohen inlen += vqp->frag_buf.npages * sizeof(__be64); 3911a86b377SEli Cohen } 3921a86b377SEli Cohen 3931a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 3941a86b377SEli Cohen if (!in) { 3951a86b377SEli Cohen err = -ENOMEM; 3961a86b377SEli Cohen goto err_kzalloc; 3971a86b377SEli Cohen } 3981a86b377SEli Cohen 3991a86b377SEli Cohen qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); 4001a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4011a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4021a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4031a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4041a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4051a86b377SEli Cohen if (!vqp->fw) 4061a86b377SEli Cohen MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma); 4071a86b377SEli Cohen MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 4081a86b377SEli Cohen err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 4091a86b377SEli Cohen kfree(in); 4101a86b377SEli Cohen if (err) 4111a86b377SEli Cohen goto err_kzalloc; 4121a86b377SEli Cohen 4131a86b377SEli Cohen vqp->mqp.uid = ndev->mvdev.res.uid; 4141a86b377SEli Cohen vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn); 4151a86b377SEli Cohen 4161a86b377SEli Cohen if (!vqp->fw) 4171a86b377SEli Cohen rx_post(vqp, mvq->num_ent); 4181a86b377SEli Cohen 4191a86b377SEli Cohen return 0; 4201a86b377SEli Cohen 4211a86b377SEli Cohen err_kzalloc: 4221a86b377SEli Cohen if (!vqp->fw) 4231a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 4241a86b377SEli Cohen err_db: 4251a86b377SEli Cohen if (!vqp->fw) 4261a86b377SEli Cohen rq_buf_free(ndev, vqp); 4271a86b377SEli Cohen 4281a86b377SEli Cohen return err; 4291a86b377SEli Cohen } 4301a86b377SEli Cohen 4311a86b377SEli Cohen static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 4321a86b377SEli Cohen { 4331a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 4341a86b377SEli Cohen 4351a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 4361a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn); 4371a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); 4381a86b377SEli Cohen if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) 4391a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); 4401a86b377SEli Cohen if (!vqp->fw) { 4411a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 4421a86b377SEli Cohen rq_buf_free(ndev, vqp); 4431a86b377SEli Cohen } 4441a86b377SEli Cohen } 4451a86b377SEli Cohen 4461a86b377SEli Cohen static void *next_cqe_sw(struct mlx5_vdpa_cq *cq) 4471a86b377SEli Cohen { 4481a86b377SEli Cohen return get_sw_cqe(cq, cq->mcq.cons_index); 4491a86b377SEli Cohen } 4501a86b377SEli Cohen 4511a86b377SEli Cohen static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq) 4521a86b377SEli Cohen { 4531a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 4541a86b377SEli Cohen 4551a86b377SEli Cohen cqe64 = next_cqe_sw(vcq); 4561a86b377SEli Cohen if (!cqe64) 4571a86b377SEli Cohen return -EAGAIN; 4581a86b377SEli Cohen 4591a86b377SEli Cohen vcq->mcq.cons_index++; 4601a86b377SEli Cohen return 0; 4611a86b377SEli Cohen } 4621a86b377SEli Cohen 4631a86b377SEli Cohen static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num) 4641a86b377SEli Cohen { 4651a86b377SEli Cohen mlx5_cq_set_ci(&mvq->cq.mcq); 4661a86b377SEli Cohen rx_post(&mvq->vqqp, num); 4671a86b377SEli Cohen if (mvq->event_cb.callback) 4681a86b377SEli Cohen mvq->event_cb.callback(mvq->event_cb.private); 4691a86b377SEli Cohen } 4701a86b377SEli Cohen 4711a86b377SEli Cohen static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) 4721a86b377SEli Cohen { 4731a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq); 4741a86b377SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 4751a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 4761a86b377SEli Cohen int num = 0; 4771a86b377SEli Cohen 4781a86b377SEli Cohen while (!mlx5_vdpa_poll_one(&mvq->cq)) { 4791a86b377SEli Cohen num++; 4801a86b377SEli Cohen if (num > mvq->num_ent / 2) { 4811a86b377SEli Cohen /* If completions keep coming while we poll, we want to 4821a86b377SEli Cohen * let the hardware know that we consumed them by 4831a86b377SEli Cohen * updating the doorbell record. We also let vdpa core 4841a86b377SEli Cohen * know about this so it passes it on the virtio driver 4851a86b377SEli Cohen * on the guest. 4861a86b377SEli Cohen */ 4871a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 4881a86b377SEli Cohen num = 0; 4891a86b377SEli Cohen } 4901a86b377SEli Cohen } 4911a86b377SEli Cohen 4921a86b377SEli Cohen if (num) 4931a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 4941a86b377SEli Cohen 4951a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 4961a86b377SEli Cohen } 4971a86b377SEli Cohen 4981a86b377SEli Cohen static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) 4991a86b377SEli Cohen { 5001a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 5011a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 5021a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5031a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 5041a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 5051a86b377SEli Cohen unsigned int irqn; 5061a86b377SEli Cohen __be64 *pas; 5071a86b377SEli Cohen int inlen; 5081a86b377SEli Cohen void *cqc; 5091a86b377SEli Cohen void *in; 5101a86b377SEli Cohen int err; 5111a86b377SEli Cohen int eqn; 5121a86b377SEli Cohen 5131a86b377SEli Cohen err = mlx5_db_alloc(mdev, &vcq->db); 5141a86b377SEli Cohen if (err) 5151a86b377SEli Cohen return err; 5161a86b377SEli Cohen 5171a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 5181a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 5191a86b377SEli Cohen vcq->mcq.cqe_sz = 64; 5201a86b377SEli Cohen 5211a86b377SEli Cohen err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); 5221a86b377SEli Cohen if (err) 5231a86b377SEli Cohen goto err_db; 5241a86b377SEli Cohen 5251a86b377SEli Cohen cq_frag_buf_init(vcq, &vcq->buf); 5261a86b377SEli Cohen 5271a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 5281a86b377SEli Cohen MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages; 5291a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 5301a86b377SEli Cohen if (!in) { 5311a86b377SEli Cohen err = -ENOMEM; 5321a86b377SEli Cohen goto err_vzalloc; 5331a86b377SEli Cohen } 5341a86b377SEli Cohen 5351a86b377SEli Cohen MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); 5361a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 5371a86b377SEli Cohen mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas); 5381a86b377SEli Cohen 5391a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 5401a86b377SEli Cohen MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 5411a86b377SEli Cohen 5421a86b377SEli Cohen /* Use vector 0 by default. Consider adding code to choose least used 5431a86b377SEli Cohen * vector. 5441a86b377SEli Cohen */ 5451a86b377SEli Cohen err = mlx5_vector2eqn(mdev, 0, &eqn, &irqn); 5461a86b377SEli Cohen if (err) 5471a86b377SEli Cohen goto err_vec; 5481a86b377SEli Cohen 5491a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 5501a86b377SEli Cohen MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent)); 5511a86b377SEli Cohen MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); 5521a86b377SEli Cohen MLX5_SET(cqc, cqc, c_eqn, eqn); 5531a86b377SEli Cohen MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma); 5541a86b377SEli Cohen 5551a86b377SEli Cohen err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out)); 5561a86b377SEli Cohen if (err) 5571a86b377SEli Cohen goto err_vec; 5581a86b377SEli Cohen 5591a86b377SEli Cohen vcq->mcq.comp = mlx5_vdpa_cq_comp; 5601a86b377SEli Cohen vcq->cqe = num_ent; 5611a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 5621a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 5631a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 5641a86b377SEli Cohen kfree(in); 5651a86b377SEli Cohen return 0; 5661a86b377SEli Cohen 5671a86b377SEli Cohen err_vec: 5681a86b377SEli Cohen kfree(in); 5691a86b377SEli Cohen err_vzalloc: 5701a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 5711a86b377SEli Cohen err_db: 5721a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 5731a86b377SEli Cohen return err; 5741a86b377SEli Cohen } 5751a86b377SEli Cohen 5761a86b377SEli Cohen static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) 5771a86b377SEli Cohen { 5781a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 5791a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 5801a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 5811a86b377SEli Cohen 5821a86b377SEli Cohen if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) { 5831a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); 5841a86b377SEli Cohen return; 5851a86b377SEli Cohen } 5861a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 5871a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 5881a86b377SEli Cohen } 5891a86b377SEli Cohen 5901a86b377SEli Cohen static int umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, 5911a86b377SEli Cohen struct mlx5_vdpa_umem **umemp) 5921a86b377SEli Cohen { 5931a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 5941a86b377SEli Cohen int p_a; 5951a86b377SEli Cohen int p_b; 5961a86b377SEli Cohen 5971a86b377SEli Cohen switch (num) { 5981a86b377SEli Cohen case 1: 5991a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a); 6001a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b); 6011a86b377SEli Cohen *umemp = &mvq->umem1; 6021a86b377SEli Cohen break; 6031a86b377SEli Cohen case 2: 6041a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a); 6051a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b); 6061a86b377SEli Cohen *umemp = &mvq->umem2; 6071a86b377SEli Cohen break; 6081a86b377SEli Cohen case 3: 6091a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a); 6101a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b); 6111a86b377SEli Cohen *umemp = &mvq->umem3; 6121a86b377SEli Cohen break; 6131a86b377SEli Cohen } 6141a86b377SEli Cohen return p_a * mvq->num_ent + p_b; 6151a86b377SEli Cohen } 6161a86b377SEli Cohen 6171a86b377SEli Cohen static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) 6181a86b377SEli Cohen { 6191a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); 6201a86b377SEli Cohen } 6211a86b377SEli Cohen 6221a86b377SEli Cohen static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 6231a86b377SEli Cohen { 6241a86b377SEli Cohen int inlen; 6251a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {}; 6261a86b377SEli Cohen void *um; 6271a86b377SEli Cohen void *in; 6281a86b377SEli Cohen int err; 6291a86b377SEli Cohen __be64 *pas; 6301a86b377SEli Cohen int size; 6311a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 6321a86b377SEli Cohen 6331a86b377SEli Cohen size = umem_size(ndev, mvq, num, &umem); 6341a86b377SEli Cohen if (size < 0) 6351a86b377SEli Cohen return size; 6361a86b377SEli Cohen 6371a86b377SEli Cohen umem->size = size; 6381a86b377SEli Cohen err = umem_frag_buf_alloc(ndev, umem, size); 6391a86b377SEli Cohen if (err) 6401a86b377SEli Cohen return err; 6411a86b377SEli Cohen 6421a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages; 6431a86b377SEli Cohen 6441a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 6451a86b377SEli Cohen if (!in) { 6461a86b377SEli Cohen err = -ENOMEM; 6471a86b377SEli Cohen goto err_in; 6481a86b377SEli Cohen } 6491a86b377SEli Cohen 6501a86b377SEli Cohen MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM); 6511a86b377SEli Cohen MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); 6521a86b377SEli Cohen um = MLX5_ADDR_OF(create_umem_in, in, umem); 6531a86b377SEli Cohen MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 6541a86b377SEli Cohen MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages); 6551a86b377SEli Cohen 6561a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]); 6571a86b377SEli Cohen mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW); 6581a86b377SEli Cohen 6591a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 6601a86b377SEli Cohen if (err) { 6611a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); 6621a86b377SEli Cohen goto err_cmd; 6631a86b377SEli Cohen } 6641a86b377SEli Cohen 6651a86b377SEli Cohen kfree(in); 6661a86b377SEli Cohen umem->id = MLX5_GET(create_umem_out, out, umem_id); 6671a86b377SEli Cohen 6681a86b377SEli Cohen return 0; 6691a86b377SEli Cohen 6701a86b377SEli Cohen err_cmd: 6711a86b377SEli Cohen kfree(in); 6721a86b377SEli Cohen err_in: 6731a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 6741a86b377SEli Cohen return err; 6751a86b377SEli Cohen } 6761a86b377SEli Cohen 6771a86b377SEli Cohen static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 6781a86b377SEli Cohen { 6791a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {}; 6801a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {}; 6811a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 6821a86b377SEli Cohen 6831a86b377SEli Cohen switch (num) { 6841a86b377SEli Cohen case 1: 6851a86b377SEli Cohen umem = &mvq->umem1; 6861a86b377SEli Cohen break; 6871a86b377SEli Cohen case 2: 6881a86b377SEli Cohen umem = &mvq->umem2; 6891a86b377SEli Cohen break; 6901a86b377SEli Cohen case 3: 6911a86b377SEli Cohen umem = &mvq->umem3; 6921a86b377SEli Cohen break; 6931a86b377SEli Cohen } 6941a86b377SEli Cohen 6951a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM); 6961a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, umem_id, umem->id); 6971a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 6981a86b377SEli Cohen return; 6991a86b377SEli Cohen 7001a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7011a86b377SEli Cohen } 7021a86b377SEli Cohen 7031a86b377SEli Cohen static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7041a86b377SEli Cohen { 7051a86b377SEli Cohen int num; 7061a86b377SEli Cohen int err; 7071a86b377SEli Cohen 7081a86b377SEli Cohen for (num = 1; num <= 3; num++) { 7091a86b377SEli Cohen err = create_umem(ndev, mvq, num); 7101a86b377SEli Cohen if (err) 7111a86b377SEli Cohen goto err_umem; 7121a86b377SEli Cohen } 7131a86b377SEli Cohen return 0; 7141a86b377SEli Cohen 7151a86b377SEli Cohen err_umem: 7161a86b377SEli Cohen for (num--; num > 0; num--) 7171a86b377SEli Cohen umem_destroy(ndev, mvq, num); 7181a86b377SEli Cohen 7191a86b377SEli Cohen return err; 7201a86b377SEli Cohen } 7211a86b377SEli Cohen 7221a86b377SEli Cohen static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7231a86b377SEli Cohen { 7241a86b377SEli Cohen int num; 7251a86b377SEli Cohen 7261a86b377SEli Cohen for (num = 3; num > 0; num--) 7271a86b377SEli Cohen umem_destroy(ndev, mvq, num); 7281a86b377SEli Cohen } 7291a86b377SEli Cohen 7301a86b377SEli Cohen static int get_queue_type(struct mlx5_vdpa_net *ndev) 7311a86b377SEli Cohen { 7321a86b377SEli Cohen u32 type_mask; 7331a86b377SEli Cohen 7341a86b377SEli Cohen type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); 7351a86b377SEli Cohen 7361a86b377SEli Cohen /* prefer split queue */ 7371a86b377SEli Cohen if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED) 7381a86b377SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED; 7391a86b377SEli Cohen 7401a86b377SEli Cohen WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)); 7411a86b377SEli Cohen 7421a86b377SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT; 7431a86b377SEli Cohen } 7441a86b377SEli Cohen 7451a86b377SEli Cohen static bool vq_is_tx(u16 idx) 7461a86b377SEli Cohen { 7471a86b377SEli Cohen return idx % 2; 7481a86b377SEli Cohen } 7491a86b377SEli Cohen 7501a86b377SEli Cohen static u16 get_features_12_3(u64 features) 7511a86b377SEli Cohen { 7521a86b377SEli Cohen return (!!(features & BIT(VIRTIO_NET_F_HOST_TSO4)) << 9) | 7531a86b377SEli Cohen (!!(features & BIT(VIRTIO_NET_F_HOST_TSO6)) << 8) | 7541a86b377SEli Cohen (!!(features & BIT(VIRTIO_NET_F_CSUM)) << 7) | 7551a86b377SEli Cohen (!!(features & BIT(VIRTIO_NET_F_GUEST_CSUM)) << 6); 7561a86b377SEli Cohen } 7571a86b377SEli Cohen 7581a86b377SEli Cohen static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7591a86b377SEli Cohen { 7601a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); 7611a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {}; 7621a86b377SEli Cohen void *obj_context; 7631a86b377SEli Cohen void *cmd_hdr; 7641a86b377SEli Cohen void *vq_ctx; 7651a86b377SEli Cohen void *in; 7661a86b377SEli Cohen int err; 7671a86b377SEli Cohen 7681a86b377SEli Cohen err = umems_create(ndev, mvq); 7691a86b377SEli Cohen if (err) 7701a86b377SEli Cohen return err; 7711a86b377SEli Cohen 7721a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 7731a86b377SEli Cohen if (!in) { 7741a86b377SEli Cohen err = -ENOMEM; 7751a86b377SEli Cohen goto err_alloc; 7761a86b377SEli Cohen } 7771a86b377SEli Cohen 7781a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr); 7791a86b377SEli Cohen 7801a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 7811a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 7821a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 7831a86b377SEli Cohen 7841a86b377SEli Cohen obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context); 7851a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); 7861a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, 7871a86b377SEli Cohen get_features_12_3(ndev->mvdev.actual_features)); 7881a86b377SEli Cohen vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context); 7891a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); 7901a86b377SEli Cohen 7911a86b377SEli Cohen if (vq_is_tx(mvq->index)) 7921a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); 7931a86b377SEli Cohen 7941a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 7951a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); 7961a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 7971a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); 7981a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 7991a86b377SEli Cohen !!(ndev->mvdev.actual_features & VIRTIO_F_VERSION_1)); 8001a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); 8011a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); 8021a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); 8031a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey.key); 8041a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); 8051a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); 8061a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id); 8071a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem1.size); 8081a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id); 8091a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem1.size); 8101a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); 8111a86b377SEli Cohen if (MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, eth_frame_offload_type)) 8121a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 1); 8131a86b377SEli Cohen 8141a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 8151a86b377SEli Cohen if (err) 8161a86b377SEli Cohen goto err_cmd; 8171a86b377SEli Cohen 8181a86b377SEli Cohen kfree(in); 8191a86b377SEli Cohen mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 8201a86b377SEli Cohen 8211a86b377SEli Cohen return 0; 8221a86b377SEli Cohen 8231a86b377SEli Cohen err_cmd: 8241a86b377SEli Cohen kfree(in); 8251a86b377SEli Cohen err_alloc: 8261a86b377SEli Cohen umems_destroy(ndev, mvq); 8271a86b377SEli Cohen return err; 8281a86b377SEli Cohen } 8291a86b377SEli Cohen 8301a86b377SEli Cohen static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 8311a86b377SEli Cohen { 8321a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {}; 8331a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {}; 8341a86b377SEli Cohen 8351a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode, 8361a86b377SEli Cohen MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 8371a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id); 8381a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); 8391a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type, 8401a86b377SEli Cohen MLX5_OBJ_TYPE_VIRTIO_NET_Q); 8411a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { 8421a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); 8431a86b377SEli Cohen return; 8441a86b377SEli Cohen } 8451a86b377SEli Cohen umems_destroy(ndev, mvq); 8461a86b377SEli Cohen } 8471a86b377SEli Cohen 8481a86b377SEli Cohen static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 8491a86b377SEli Cohen { 8501a86b377SEli Cohen return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn; 8511a86b377SEli Cohen } 8521a86b377SEli Cohen 8531a86b377SEli Cohen static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 8541a86b377SEli Cohen { 8551a86b377SEli Cohen return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn; 8561a86b377SEli Cohen } 8571a86b377SEli Cohen 8581a86b377SEli Cohen static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, 8591a86b377SEli Cohen int *outlen, u32 qpn, u32 rqpn) 8601a86b377SEli Cohen { 8611a86b377SEli Cohen void *qpc; 8621a86b377SEli Cohen void *pp; 8631a86b377SEli Cohen 8641a86b377SEli Cohen switch (cmd) { 8651a86b377SEli Cohen case MLX5_CMD_OP_2RST_QP: 8661a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in); 8671a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out); 8681a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 8691a86b377SEli Cohen *out = kzalloc(*outlen, GFP_KERNEL); 870f31231bfSColin Ian King if (!*in || !*out) 8711a86b377SEli Cohen goto outerr; 8721a86b377SEli Cohen 8731a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, opcode, cmd); 8741a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); 8751a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, qpn, qpn); 8761a86b377SEli Cohen break; 8771a86b377SEli Cohen case MLX5_CMD_OP_RST2INIT_QP: 8781a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in); 8791a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out); 8801a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 8811a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL); 882f31231bfSColin Ian King if (!*in || !*out) 8831a86b377SEli Cohen goto outerr; 8841a86b377SEli Cohen 8851a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, opcode, cmd); 8861a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); 8871a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, qpn, qpn); 8881a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 8891a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 8901a86b377SEli Cohen MLX5_SET(qpc, qpc, rwe, 1); 8911a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 8921a86b377SEli Cohen MLX5_SET(ads, pp, vhca_port_num, 1); 8931a86b377SEli Cohen break; 8941a86b377SEli Cohen case MLX5_CMD_OP_INIT2RTR_QP: 8951a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in); 8961a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out); 8971a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 8981a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL); 899f31231bfSColin Ian King if (!*in || !*out) 9001a86b377SEli Cohen goto outerr; 9011a86b377SEli Cohen 9021a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, opcode, cmd); 9031a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); 9041a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, qpn, qpn); 9051a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9061a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 9071a86b377SEli Cohen MLX5_SET(qpc, qpc, log_msg_max, 30); 9081a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9091a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9101a86b377SEli Cohen MLX5_SET(ads, pp, fl, 1); 9111a86b377SEli Cohen break; 9121a86b377SEli Cohen case MLX5_CMD_OP_RTR2RTS_QP: 9131a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in); 9141a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out); 9151a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9161a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL); 917f31231bfSColin Ian King if (!*in || !*out) 9181a86b377SEli Cohen goto outerr; 9191a86b377SEli Cohen 9201a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd); 9211a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); 9221a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn); 9231a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9241a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9251a86b377SEli Cohen MLX5_SET(ads, pp, ack_timeout, 14); 9261a86b377SEli Cohen MLX5_SET(qpc, qpc, retry_count, 7); 9271a86b377SEli Cohen MLX5_SET(qpc, qpc, rnr_retry, 7); 9281a86b377SEli Cohen break; 9291a86b377SEli Cohen default: 930f31231bfSColin Ian King goto outerr_nullify; 9311a86b377SEli Cohen } 9321a86b377SEli Cohen 9331a86b377SEli Cohen return; 9341a86b377SEli Cohen 9351a86b377SEli Cohen outerr: 9361a86b377SEli Cohen kfree(*in); 9371a86b377SEli Cohen kfree(*out); 938f31231bfSColin Ian King outerr_nullify: 9391a86b377SEli Cohen *in = NULL; 9401a86b377SEli Cohen *out = NULL; 9411a86b377SEli Cohen } 9421a86b377SEli Cohen 9431a86b377SEli Cohen static void free_inout(void *in, void *out) 9441a86b377SEli Cohen { 9451a86b377SEli Cohen kfree(in); 9461a86b377SEli Cohen kfree(out); 9471a86b377SEli Cohen } 9481a86b377SEli Cohen 9491a86b377SEli Cohen /* Two QPs are used by each virtqueue. One is used by the driver and one by 9501a86b377SEli Cohen * firmware. The fw argument indicates whether the subjected QP is the one used 9511a86b377SEli Cohen * by firmware. 9521a86b377SEli Cohen */ 9531a86b377SEli Cohen static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) 9541a86b377SEli Cohen { 9551a86b377SEli Cohen int outlen; 9561a86b377SEli Cohen int inlen; 9571a86b377SEli Cohen void *out; 9581a86b377SEli Cohen void *in; 9591a86b377SEli Cohen int err; 9601a86b377SEli Cohen 9611a86b377SEli Cohen alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); 9621a86b377SEli Cohen if (!in || !out) 9631a86b377SEli Cohen return -ENOMEM; 9641a86b377SEli Cohen 9651a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); 9661a86b377SEli Cohen free_inout(in, out); 9671a86b377SEli Cohen return err; 9681a86b377SEli Cohen } 9691a86b377SEli Cohen 9701a86b377SEli Cohen static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 9711a86b377SEli Cohen { 9721a86b377SEli Cohen int err; 9731a86b377SEli Cohen 9741a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); 9751a86b377SEli Cohen if (err) 9761a86b377SEli Cohen return err; 9771a86b377SEli Cohen 9781a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); 9791a86b377SEli Cohen if (err) 9801a86b377SEli Cohen return err; 9811a86b377SEli Cohen 9821a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); 9831a86b377SEli Cohen if (err) 9841a86b377SEli Cohen return err; 9851a86b377SEli Cohen 9861a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); 9871a86b377SEli Cohen if (err) 9881a86b377SEli Cohen return err; 9891a86b377SEli Cohen 9901a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); 9911a86b377SEli Cohen if (err) 9921a86b377SEli Cohen return err; 9931a86b377SEli Cohen 9941a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); 9951a86b377SEli Cohen if (err) 9961a86b377SEli Cohen return err; 9971a86b377SEli Cohen 9981a86b377SEli Cohen return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); 9991a86b377SEli Cohen } 10001a86b377SEli Cohen 10011a86b377SEli Cohen struct mlx5_virtq_attr { 10021a86b377SEli Cohen u8 state; 10031a86b377SEli Cohen u16 available_index; 10041a86b377SEli Cohen }; 10051a86b377SEli Cohen 10061a86b377SEli Cohen static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 10071a86b377SEli Cohen struct mlx5_virtq_attr *attr) 10081a86b377SEli Cohen { 10091a86b377SEli Cohen int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out); 10101a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {}; 10111a86b377SEli Cohen void *out; 10121a86b377SEli Cohen void *obj_context; 10131a86b377SEli Cohen void *cmd_hdr; 10141a86b377SEli Cohen int err; 10151a86b377SEli Cohen 10161a86b377SEli Cohen out = kzalloc(outlen, GFP_KERNEL); 10171a86b377SEli Cohen if (!out) 10181a86b377SEli Cohen return -ENOMEM; 10191a86b377SEli Cohen 10201a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr); 10211a86b377SEli Cohen 10221a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 10231a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 10241a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 10251a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 10261a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen); 10271a86b377SEli Cohen if (err) 10281a86b377SEli Cohen goto err_cmd; 10291a86b377SEli Cohen 10301a86b377SEli Cohen obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context); 10311a86b377SEli Cohen memset(attr, 0, sizeof(*attr)); 10321a86b377SEli Cohen attr->state = MLX5_GET(virtio_net_q_object, obj_context, state); 10331a86b377SEli Cohen attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index); 10341a86b377SEli Cohen kfree(out); 10351a86b377SEli Cohen return 0; 10361a86b377SEli Cohen 10371a86b377SEli Cohen err_cmd: 10381a86b377SEli Cohen kfree(out); 10391a86b377SEli Cohen return err; 10401a86b377SEli Cohen } 10411a86b377SEli Cohen 10421a86b377SEli Cohen static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) 10431a86b377SEli Cohen { 10441a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); 10451a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {}; 10461a86b377SEli Cohen void *obj_context; 10471a86b377SEli Cohen void *cmd_hdr; 10481a86b377SEli Cohen void *in; 10491a86b377SEli Cohen int err; 10501a86b377SEli Cohen 10511a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 10521a86b377SEli Cohen if (!in) 10531a86b377SEli Cohen return -ENOMEM; 10541a86b377SEli Cohen 10551a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr); 10561a86b377SEli Cohen 10571a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); 10581a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 10591a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 10601a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 10611a86b377SEli Cohen 10621a86b377SEli Cohen obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context); 10631a86b377SEli Cohen MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select, 10641a86b377SEli Cohen MLX5_VIRTQ_MODIFY_MASK_STATE); 10651a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, state, state); 10661a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 10671a86b377SEli Cohen kfree(in); 10681a86b377SEli Cohen if (!err) 10691a86b377SEli Cohen mvq->fw_state = state; 10701a86b377SEli Cohen 10711a86b377SEli Cohen return err; 10721a86b377SEli Cohen } 10731a86b377SEli Cohen 10741a86b377SEli Cohen static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 10751a86b377SEli Cohen { 10761a86b377SEli Cohen u16 idx = mvq->index; 10771a86b377SEli Cohen int err; 10781a86b377SEli Cohen 10791a86b377SEli Cohen if (!mvq->num_ent) 10801a86b377SEli Cohen return 0; 10811a86b377SEli Cohen 10821a86b377SEli Cohen if (mvq->initialized) { 10831a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "attempt re init\n"); 10841a86b377SEli Cohen return -EINVAL; 10851a86b377SEli Cohen } 10861a86b377SEli Cohen 10871a86b377SEli Cohen err = cq_create(ndev, idx, mvq->num_ent); 10881a86b377SEli Cohen if (err) 10891a86b377SEli Cohen return err; 10901a86b377SEli Cohen 10911a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->fwqp); 10921a86b377SEli Cohen if (err) 10931a86b377SEli Cohen goto err_fwqp; 10941a86b377SEli Cohen 10951a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->vqqp); 10961a86b377SEli Cohen if (err) 10971a86b377SEli Cohen goto err_vqqp; 10981a86b377SEli Cohen 10991a86b377SEli Cohen err = connect_qps(ndev, mvq); 11001a86b377SEli Cohen if (err) 11011a86b377SEli Cohen goto err_connect; 11021a86b377SEli Cohen 11031a86b377SEli Cohen err = create_virtqueue(ndev, mvq); 11041a86b377SEli Cohen if (err) 11051a86b377SEli Cohen goto err_connect; 11061a86b377SEli Cohen 11071a86b377SEli Cohen if (mvq->ready) { 11081a86b377SEli Cohen err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 11091a86b377SEli Cohen if (err) { 11101a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", 11111a86b377SEli Cohen idx, err); 11121a86b377SEli Cohen goto err_connect; 11131a86b377SEli Cohen } 11141a86b377SEli Cohen } 11151a86b377SEli Cohen 11161a86b377SEli Cohen mvq->initialized = true; 11171a86b377SEli Cohen return 0; 11181a86b377SEli Cohen 11191a86b377SEli Cohen err_connect: 11201a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 11211a86b377SEli Cohen err_vqqp: 11221a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 11231a86b377SEli Cohen err_fwqp: 11241a86b377SEli Cohen cq_destroy(ndev, idx); 11251a86b377SEli Cohen return err; 11261a86b377SEli Cohen } 11271a86b377SEli Cohen 11281a86b377SEli Cohen static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11291a86b377SEli Cohen { 11301a86b377SEli Cohen struct mlx5_virtq_attr attr; 11311a86b377SEli Cohen 11321a86b377SEli Cohen if (!mvq->initialized) 11331a86b377SEli Cohen return; 11341a86b377SEli Cohen 11351a86b377SEli Cohen if (query_virtqueue(ndev, mvq, &attr)) { 11361a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); 11371a86b377SEli Cohen return; 11381a86b377SEli Cohen } 11391a86b377SEli Cohen if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 11401a86b377SEli Cohen return; 11411a86b377SEli Cohen 11421a86b377SEli Cohen if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) 11431a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); 11441a86b377SEli Cohen } 11451a86b377SEli Cohen 11461a86b377SEli Cohen static void suspend_vqs(struct mlx5_vdpa_net *ndev) 11471a86b377SEli Cohen { 11481a86b377SEli Cohen int i; 11491a86b377SEli Cohen 11501a86b377SEli Cohen for (i = 0; i < MLX5_MAX_SUPPORTED_VQS; i++) 11511a86b377SEli Cohen suspend_vq(ndev, &ndev->vqs[i]); 11521a86b377SEli Cohen } 11531a86b377SEli Cohen 11541a86b377SEli Cohen static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11551a86b377SEli Cohen { 11561a86b377SEli Cohen if (!mvq->initialized) 11571a86b377SEli Cohen return; 11581a86b377SEli Cohen 11591a86b377SEli Cohen suspend_vq(ndev, mvq); 11601a86b377SEli Cohen destroy_virtqueue(ndev, mvq); 11611a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 11621a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 11631a86b377SEli Cohen cq_destroy(ndev, mvq->index); 11641a86b377SEli Cohen mvq->initialized = false; 11651a86b377SEli Cohen } 11661a86b377SEli Cohen 11671a86b377SEli Cohen static int create_rqt(struct mlx5_vdpa_net *ndev) 11681a86b377SEli Cohen { 11691a86b377SEli Cohen int log_max_rqt; 11701a86b377SEli Cohen __be32 *list; 11711a86b377SEli Cohen void *rqtc; 11721a86b377SEli Cohen int inlen; 11731a86b377SEli Cohen void *in; 11741a86b377SEli Cohen int i, j; 11751a86b377SEli Cohen int err; 11761a86b377SEli Cohen 11771a86b377SEli Cohen log_max_rqt = min_t(int, 1, MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size)); 11781a86b377SEli Cohen if (log_max_rqt < 1) 11791a86b377SEli Cohen return -EOPNOTSUPP; 11801a86b377SEli Cohen 11811a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + (1 << log_max_rqt) * MLX5_ST_SZ_BYTES(rq_num); 11821a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 11831a86b377SEli Cohen if (!in) 11841a86b377SEli Cohen return -ENOMEM; 11851a86b377SEli Cohen 11861a86b377SEli Cohen MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); 11871a86b377SEli Cohen rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 11881a86b377SEli Cohen 11891a86b377SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 11901a86b377SEli Cohen MLX5_SET(rqtc, rqtc, rqt_max_size, 1 << log_max_rqt); 11911a86b377SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, 1); 11921a86b377SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 11931a86b377SEli Cohen for (i = 0, j = 0; j < ndev->mvdev.max_vqs; j++) { 11941a86b377SEli Cohen if (!ndev->vqs[j].initialized) 11951a86b377SEli Cohen continue; 11961a86b377SEli Cohen 11971a86b377SEli Cohen if (!vq_is_tx(ndev->vqs[j].index)) { 11981a86b377SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j].virtq_id); 11991a86b377SEli Cohen i++; 12001a86b377SEli Cohen } 12011a86b377SEli Cohen } 12021a86b377SEli Cohen 12031a86b377SEli Cohen err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); 12041a86b377SEli Cohen kfree(in); 12051a86b377SEli Cohen if (err) 12061a86b377SEli Cohen return err; 12071a86b377SEli Cohen 12081a86b377SEli Cohen return 0; 12091a86b377SEli Cohen } 12101a86b377SEli Cohen 12111a86b377SEli Cohen static void destroy_rqt(struct mlx5_vdpa_net *ndev) 12121a86b377SEli Cohen { 12131a86b377SEli Cohen mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); 12141a86b377SEli Cohen } 12151a86b377SEli Cohen 12161a86b377SEli Cohen static int create_tir(struct mlx5_vdpa_net *ndev) 12171a86b377SEli Cohen { 12181a86b377SEli Cohen #define HASH_IP_L4PORTS \ 12191a86b377SEli Cohen (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \ 12201a86b377SEli Cohen MLX5_HASH_FIELD_SEL_L4_DPORT) 12211a86b377SEli Cohen static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 12221a86b377SEli Cohen 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 12231a86b377SEli Cohen 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 12241a86b377SEli Cohen 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 12251a86b377SEli Cohen 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a }; 12261a86b377SEli Cohen void *rss_key; 12271a86b377SEli Cohen void *outer; 12281a86b377SEli Cohen void *tirc; 12291a86b377SEli Cohen void *in; 12301a86b377SEli Cohen int err; 12311a86b377SEli Cohen 12321a86b377SEli Cohen in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL); 12331a86b377SEli Cohen if (!in) 12341a86b377SEli Cohen return -ENOMEM; 12351a86b377SEli Cohen 12361a86b377SEli Cohen MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); 12371a86b377SEli Cohen tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 12381a86b377SEli Cohen MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 12391a86b377SEli Cohen 12401a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 12411a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ); 12421a86b377SEli Cohen rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 12431a86b377SEli Cohen memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key)); 12441a86b377SEli Cohen 12451a86b377SEli Cohen outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 12461a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); 12471a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); 12481a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS); 12491a86b377SEli Cohen 12501a86b377SEli Cohen MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); 12511a86b377SEli Cohen MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); 12521a86b377SEli Cohen 12531a86b377SEli Cohen err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); 12541a86b377SEli Cohen kfree(in); 12551a86b377SEli Cohen return err; 12561a86b377SEli Cohen } 12571a86b377SEli Cohen 12581a86b377SEli Cohen static void destroy_tir(struct mlx5_vdpa_net *ndev) 12591a86b377SEli Cohen { 12601a86b377SEli Cohen mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); 12611a86b377SEli Cohen } 12621a86b377SEli Cohen 12631a86b377SEli Cohen static int add_fwd_to_tir(struct mlx5_vdpa_net *ndev) 12641a86b377SEli Cohen { 12651a86b377SEli Cohen struct mlx5_flow_destination dest[2] = {}; 12661a86b377SEli Cohen struct mlx5_flow_table_attr ft_attr = {}; 12671a86b377SEli Cohen struct mlx5_flow_act flow_act = {}; 12681a86b377SEli Cohen struct mlx5_flow_namespace *ns; 12691a86b377SEli Cohen int err; 12701a86b377SEli Cohen 12711a86b377SEli Cohen /* for now, one entry, match all, forward to tir */ 12721a86b377SEli Cohen ft_attr.max_fte = 1; 12731a86b377SEli Cohen ft_attr.autogroup.max_num_groups = 1; 12741a86b377SEli Cohen 12751a86b377SEli Cohen ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); 12761a86b377SEli Cohen if (!ns) { 12771a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "get flow namespace\n"); 12781a86b377SEli Cohen return -EOPNOTSUPP; 12791a86b377SEli Cohen } 12801a86b377SEli Cohen 12811a86b377SEli Cohen ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 12821a86b377SEli Cohen if (IS_ERR(ndev->rxft)) 12831a86b377SEli Cohen return PTR_ERR(ndev->rxft); 12841a86b377SEli Cohen 12851a86b377SEli Cohen ndev->rx_counter = mlx5_fc_create(ndev->mvdev.mdev, false); 12861a86b377SEli Cohen if (IS_ERR(ndev->rx_counter)) { 12871a86b377SEli Cohen err = PTR_ERR(ndev->rx_counter); 12881a86b377SEli Cohen goto err_fc; 12891a86b377SEli Cohen } 12901a86b377SEli Cohen 12911a86b377SEli Cohen flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_COUNT; 12921a86b377SEli Cohen dest[0].type = MLX5_FLOW_DESTINATION_TYPE_TIR; 12931a86b377SEli Cohen dest[0].tir_num = ndev->res.tirn; 12941a86b377SEli Cohen dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 12951a86b377SEli Cohen dest[1].counter_id = mlx5_fc_id(ndev->rx_counter); 12961a86b377SEli Cohen ndev->rx_rule = mlx5_add_flow_rules(ndev->rxft, NULL, &flow_act, dest, 2); 12971a86b377SEli Cohen if (IS_ERR(ndev->rx_rule)) { 12981a86b377SEli Cohen err = PTR_ERR(ndev->rx_rule); 12991a86b377SEli Cohen ndev->rx_rule = NULL; 13001a86b377SEli Cohen goto err_rule; 13011a86b377SEli Cohen } 13021a86b377SEli Cohen 13031a86b377SEli Cohen return 0; 13041a86b377SEli Cohen 13051a86b377SEli Cohen err_rule: 13061a86b377SEli Cohen mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter); 13071a86b377SEli Cohen err_fc: 13081a86b377SEli Cohen mlx5_destroy_flow_table(ndev->rxft); 13091a86b377SEli Cohen return err; 13101a86b377SEli Cohen } 13111a86b377SEli Cohen 13121a86b377SEli Cohen static void remove_fwd_to_tir(struct mlx5_vdpa_net *ndev) 13131a86b377SEli Cohen { 13141a86b377SEli Cohen if (!ndev->rx_rule) 13151a86b377SEli Cohen return; 13161a86b377SEli Cohen 13171a86b377SEli Cohen mlx5_del_flow_rules(ndev->rx_rule); 13181a86b377SEli Cohen mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter); 13191a86b377SEli Cohen mlx5_destroy_flow_table(ndev->rxft); 13201a86b377SEli Cohen 13211a86b377SEli Cohen ndev->rx_rule = NULL; 13221a86b377SEli Cohen } 13231a86b377SEli Cohen 13241a86b377SEli Cohen static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) 13251a86b377SEli Cohen { 13261a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 13271a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 13281a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 13291a86b377SEli Cohen 13301a86b377SEli Cohen if (unlikely(!mvq->ready)) 13311a86b377SEli Cohen return; 13321a86b377SEli Cohen 13331a86b377SEli Cohen iowrite16(idx, ndev->mvdev.res.kick_addr); 13341a86b377SEli Cohen } 13351a86b377SEli Cohen 13361a86b377SEli Cohen static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 13371a86b377SEli Cohen u64 driver_area, u64 device_area) 13381a86b377SEli Cohen { 13391a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 13401a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 13411a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 13421a86b377SEli Cohen 13431a86b377SEli Cohen mvq->desc_addr = desc_area; 13441a86b377SEli Cohen mvq->device_addr = device_area; 13451a86b377SEli Cohen mvq->driver_addr = driver_area; 13461a86b377SEli Cohen return 0; 13471a86b377SEli Cohen } 13481a86b377SEli Cohen 13491a86b377SEli Cohen static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 13501a86b377SEli Cohen { 13511a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 13521a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 13531a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 13541a86b377SEli Cohen 13551a86b377SEli Cohen mvq = &ndev->vqs[idx]; 13561a86b377SEli Cohen mvq->num_ent = num; 13571a86b377SEli Cohen } 13581a86b377SEli Cohen 13591a86b377SEli Cohen static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 13601a86b377SEli Cohen { 13611a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 13621a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 13631a86b377SEli Cohen struct mlx5_vdpa_virtqueue *vq = &ndev->vqs[idx]; 13641a86b377SEli Cohen 13651a86b377SEli Cohen vq->event_cb = *cb; 13661a86b377SEli Cohen } 13671a86b377SEli Cohen 13681a86b377SEli Cohen static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 13691a86b377SEli Cohen { 13701a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 13711a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 13721a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 13731a86b377SEli Cohen 13741a86b377SEli Cohen if (!ready) 13751a86b377SEli Cohen suspend_vq(ndev, mvq); 13761a86b377SEli Cohen 13771a86b377SEli Cohen mvq->ready = ready; 13781a86b377SEli Cohen } 13791a86b377SEli Cohen 13801a86b377SEli Cohen static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx) 13811a86b377SEli Cohen { 13821a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 13831a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 13841a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 13851a86b377SEli Cohen 13861a86b377SEli Cohen return mvq->ready; 13871a86b377SEli Cohen } 13881a86b377SEli Cohen 13891a86b377SEli Cohen static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx, 13901a86b377SEli Cohen const struct vdpa_vq_state *state) 13911a86b377SEli Cohen { 13921a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 13931a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 13941a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 13951a86b377SEli Cohen 13961a86b377SEli Cohen if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) { 13971a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "can't modify available index\n"); 13981a86b377SEli Cohen return -EINVAL; 13991a86b377SEli Cohen } 14001a86b377SEli Cohen 14011a86b377SEli Cohen mvq->avail_idx = state->avail_index; 14021a86b377SEli Cohen return 0; 14031a86b377SEli Cohen } 14041a86b377SEli Cohen 14051a86b377SEli Cohen static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 14061a86b377SEli Cohen { 14071a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 14081a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 14091a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 14101a86b377SEli Cohen struct mlx5_virtq_attr attr; 14111a86b377SEli Cohen int err; 14121a86b377SEli Cohen 14131a86b377SEli Cohen if (!mvq->initialized) 14141a86b377SEli Cohen return -EAGAIN; 14151a86b377SEli Cohen 14161a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 14171a86b377SEli Cohen if (err) { 14181a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n"); 14191a86b377SEli Cohen return err; 14201a86b377SEli Cohen } 14211a86b377SEli Cohen state->avail_index = attr.available_index; 14221a86b377SEli Cohen return 0; 14231a86b377SEli Cohen } 14241a86b377SEli Cohen 14251a86b377SEli Cohen static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) 14261a86b377SEli Cohen { 14271a86b377SEli Cohen return PAGE_SIZE; 14281a86b377SEli Cohen } 14291a86b377SEli Cohen 14301a86b377SEli Cohen enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9, 14311a86b377SEli Cohen MLX5_VIRTIO_NET_F_CSUM = 1 << 10, 14321a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11, 14331a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12, 14341a86b377SEli Cohen }; 14351a86b377SEli Cohen 14361a86b377SEli Cohen static u64 mlx_to_vritio_features(u16 dev_features) 14371a86b377SEli Cohen { 14381a86b377SEli Cohen u64 result = 0; 14391a86b377SEli Cohen 14401a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM) 14411a86b377SEli Cohen result |= BIT(VIRTIO_NET_F_GUEST_CSUM); 14421a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_CSUM) 14431a86b377SEli Cohen result |= BIT(VIRTIO_NET_F_CSUM); 14441a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6) 14451a86b377SEli Cohen result |= BIT(VIRTIO_NET_F_HOST_TSO6); 14461a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4) 14471a86b377SEli Cohen result |= BIT(VIRTIO_NET_F_HOST_TSO4); 14481a86b377SEli Cohen 14491a86b377SEli Cohen return result; 14501a86b377SEli Cohen } 14511a86b377SEli Cohen 14521a86b377SEli Cohen static u64 mlx5_vdpa_get_features(struct vdpa_device *vdev) 14531a86b377SEli Cohen { 14541a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 14551a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 14561a86b377SEli Cohen u16 dev_features; 14571a86b377SEli Cohen 14581a86b377SEli Cohen dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, device_features_bits_mask); 14591a86b377SEli Cohen ndev->mvdev.mlx_features = mlx_to_vritio_features(dev_features); 14601a86b377SEli Cohen if (MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, virtio_version_1_0)) 14611a86b377SEli Cohen ndev->mvdev.mlx_features |= BIT(VIRTIO_F_VERSION_1); 14621a86b377SEli Cohen ndev->mvdev.mlx_features |= BIT(VIRTIO_F_ACCESS_PLATFORM); 14631a86b377SEli Cohen print_features(mvdev, ndev->mvdev.mlx_features, false); 14641a86b377SEli Cohen return ndev->mvdev.mlx_features; 14651a86b377SEli Cohen } 14661a86b377SEli Cohen 14671a86b377SEli Cohen static int verify_min_features(struct mlx5_vdpa_dev *mvdev, u64 features) 14681a86b377SEli Cohen { 14691a86b377SEli Cohen if (!(features & BIT(VIRTIO_F_ACCESS_PLATFORM))) 14701a86b377SEli Cohen return -EOPNOTSUPP; 14711a86b377SEli Cohen 14721a86b377SEli Cohen return 0; 14731a86b377SEli Cohen } 14741a86b377SEli Cohen 14751a86b377SEli Cohen static int setup_virtqueues(struct mlx5_vdpa_net *ndev) 14761a86b377SEli Cohen { 14771a86b377SEli Cohen int err; 14781a86b377SEli Cohen int i; 14791a86b377SEli Cohen 14801a86b377SEli Cohen for (i = 0; i < 2 * mlx5_vdpa_max_qps(ndev->mvdev.max_vqs); i++) { 14811a86b377SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 14821a86b377SEli Cohen if (err) 14831a86b377SEli Cohen goto err_vq; 14841a86b377SEli Cohen } 14851a86b377SEli Cohen 14861a86b377SEli Cohen return 0; 14871a86b377SEli Cohen 14881a86b377SEli Cohen err_vq: 14891a86b377SEli Cohen for (--i; i >= 0; i--) 14901a86b377SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 14911a86b377SEli Cohen 14921a86b377SEli Cohen return err; 14931a86b377SEli Cohen } 14941a86b377SEli Cohen 14951a86b377SEli Cohen static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) 14961a86b377SEli Cohen { 14971a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 14981a86b377SEli Cohen int i; 14991a86b377SEli Cohen 15001a86b377SEli Cohen for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { 15011a86b377SEli Cohen mvq = &ndev->vqs[i]; 15021a86b377SEli Cohen if (!mvq->initialized) 15031a86b377SEli Cohen continue; 15041a86b377SEli Cohen 15051a86b377SEli Cohen teardown_vq(ndev, mvq); 15061a86b377SEli Cohen } 15071a86b377SEli Cohen } 15081a86b377SEli Cohen 15091a86b377SEli Cohen static int mlx5_vdpa_set_features(struct vdpa_device *vdev, u64 features) 15101a86b377SEli Cohen { 15111a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 15121a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 15131a86b377SEli Cohen int err; 15141a86b377SEli Cohen 15151a86b377SEli Cohen print_features(mvdev, features, true); 15161a86b377SEli Cohen 15171a86b377SEli Cohen err = verify_min_features(mvdev, features); 15181a86b377SEli Cohen if (err) 15191a86b377SEli Cohen return err; 15201a86b377SEli Cohen 15211a86b377SEli Cohen ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; 15221a86b377SEli Cohen return err; 15231a86b377SEli Cohen } 15241a86b377SEli Cohen 15251a86b377SEli Cohen static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 15261a86b377SEli Cohen { 15271a86b377SEli Cohen /* not implemented */ 15281a86b377SEli Cohen mlx5_vdpa_warn(to_mvdev(vdev), "set config callback not supported\n"); 15291a86b377SEli Cohen } 15301a86b377SEli Cohen 15311a86b377SEli Cohen #define MLX5_VDPA_MAX_VQ_ENTRIES 256 15321a86b377SEli Cohen static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev) 15331a86b377SEli Cohen { 15341a86b377SEli Cohen return MLX5_VDPA_MAX_VQ_ENTRIES; 15351a86b377SEli Cohen } 15361a86b377SEli Cohen 15371a86b377SEli Cohen static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev) 15381a86b377SEli Cohen { 15391a86b377SEli Cohen return VIRTIO_ID_NET; 15401a86b377SEli Cohen } 15411a86b377SEli Cohen 15421a86b377SEli Cohen static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev) 15431a86b377SEli Cohen { 15441a86b377SEli Cohen return PCI_VENDOR_ID_MELLANOX; 15451a86b377SEli Cohen } 15461a86b377SEli Cohen 15471a86b377SEli Cohen static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev) 15481a86b377SEli Cohen { 15491a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 15501a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 15511a86b377SEli Cohen 15521a86b377SEli Cohen print_status(mvdev, ndev->mvdev.status, false); 15531a86b377SEli Cohen return ndev->mvdev.status; 15541a86b377SEli Cohen } 15551a86b377SEli Cohen 15561a86b377SEli Cohen static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 15571a86b377SEli Cohen { 15581a86b377SEli Cohen struct mlx5_vq_restore_info *ri = &mvq->ri; 15591a86b377SEli Cohen struct mlx5_virtq_attr attr; 15601a86b377SEli Cohen int err; 15611a86b377SEli Cohen 15621a86b377SEli Cohen if (!mvq->initialized) 15631a86b377SEli Cohen return 0; 15641a86b377SEli Cohen 15651a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 15661a86b377SEli Cohen if (err) 15671a86b377SEli Cohen return err; 15681a86b377SEli Cohen 15691a86b377SEli Cohen ri->avail_index = attr.available_index; 15701a86b377SEli Cohen ri->ready = mvq->ready; 15711a86b377SEli Cohen ri->num_ent = mvq->num_ent; 15721a86b377SEli Cohen ri->desc_addr = mvq->desc_addr; 15731a86b377SEli Cohen ri->device_addr = mvq->device_addr; 15741a86b377SEli Cohen ri->driver_addr = mvq->driver_addr; 15751a86b377SEli Cohen ri->cb = mvq->event_cb; 15761a86b377SEli Cohen ri->restore = true; 15771a86b377SEli Cohen return 0; 15781a86b377SEli Cohen } 15791a86b377SEli Cohen 15801a86b377SEli Cohen static int save_channels_info(struct mlx5_vdpa_net *ndev) 15811a86b377SEli Cohen { 15821a86b377SEli Cohen int i; 15831a86b377SEli Cohen 15841a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 15851a86b377SEli Cohen memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); 15861a86b377SEli Cohen save_channel_info(ndev, &ndev->vqs[i]); 15871a86b377SEli Cohen } 15881a86b377SEli Cohen return 0; 15891a86b377SEli Cohen } 15901a86b377SEli Cohen 15911a86b377SEli Cohen static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) 15921a86b377SEli Cohen { 15931a86b377SEli Cohen int i; 15941a86b377SEli Cohen 15951a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 15961a86b377SEli Cohen memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 15971a86b377SEli Cohen } 15981a86b377SEli Cohen 15991a86b377SEli Cohen static void restore_channels_info(struct mlx5_vdpa_net *ndev) 16001a86b377SEli Cohen { 16011a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 16021a86b377SEli Cohen struct mlx5_vq_restore_info *ri; 16031a86b377SEli Cohen int i; 16041a86b377SEli Cohen 16051a86b377SEli Cohen mlx5_clear_vqs(ndev); 16061a86b377SEli Cohen init_mvqs(ndev); 16071a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 16081a86b377SEli Cohen mvq = &ndev->vqs[i]; 16091a86b377SEli Cohen ri = &mvq->ri; 16101a86b377SEli Cohen if (!ri->restore) 16111a86b377SEli Cohen continue; 16121a86b377SEli Cohen 16131a86b377SEli Cohen mvq->avail_idx = ri->avail_index; 16141a86b377SEli Cohen mvq->ready = ri->ready; 16151a86b377SEli Cohen mvq->num_ent = ri->num_ent; 16161a86b377SEli Cohen mvq->desc_addr = ri->desc_addr; 16171a86b377SEli Cohen mvq->device_addr = ri->device_addr; 16181a86b377SEli Cohen mvq->driver_addr = ri->driver_addr; 16191a86b377SEli Cohen mvq->event_cb = ri->cb; 16201a86b377SEli Cohen } 16211a86b377SEli Cohen } 16221a86b377SEli Cohen 16231a86b377SEli Cohen static int mlx5_vdpa_change_map(struct mlx5_vdpa_net *ndev, struct vhost_iotlb *iotlb) 16241a86b377SEli Cohen { 16251a86b377SEli Cohen int err; 16261a86b377SEli Cohen 16271a86b377SEli Cohen suspend_vqs(ndev); 16281a86b377SEli Cohen err = save_channels_info(ndev); 16291a86b377SEli Cohen if (err) 16301a86b377SEli Cohen goto err_mr; 16311a86b377SEli Cohen 16321a86b377SEli Cohen teardown_driver(ndev); 16331a86b377SEli Cohen mlx5_vdpa_destroy_mr(&ndev->mvdev); 16341a86b377SEli Cohen err = mlx5_vdpa_create_mr(&ndev->mvdev, iotlb); 16351a86b377SEli Cohen if (err) 16361a86b377SEli Cohen goto err_mr; 16371a86b377SEli Cohen 16381a86b377SEli Cohen restore_channels_info(ndev); 16391a86b377SEli Cohen err = setup_driver(ndev); 16401a86b377SEli Cohen if (err) 16411a86b377SEli Cohen goto err_setup; 16421a86b377SEli Cohen 16431a86b377SEli Cohen return 0; 16441a86b377SEli Cohen 16451a86b377SEli Cohen err_setup: 16461a86b377SEli Cohen mlx5_vdpa_destroy_mr(&ndev->mvdev); 16471a86b377SEli Cohen err_mr: 16481a86b377SEli Cohen return err; 16491a86b377SEli Cohen } 16501a86b377SEli Cohen 16511a86b377SEli Cohen static int setup_driver(struct mlx5_vdpa_net *ndev) 16521a86b377SEli Cohen { 16531a86b377SEli Cohen int err; 16541a86b377SEli Cohen 16551a86b377SEli Cohen mutex_lock(&ndev->reslock); 16561a86b377SEli Cohen if (ndev->setup) { 16571a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "setup driver called for already setup driver\n"); 16581a86b377SEli Cohen err = 0; 16591a86b377SEli Cohen goto out; 16601a86b377SEli Cohen } 16611a86b377SEli Cohen err = setup_virtqueues(ndev); 16621a86b377SEli Cohen if (err) { 16631a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "setup_virtqueues\n"); 16641a86b377SEli Cohen goto out; 16651a86b377SEli Cohen } 16661a86b377SEli Cohen 16671a86b377SEli Cohen err = create_rqt(ndev); 16681a86b377SEli Cohen if (err) { 16691a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "create_rqt\n"); 16701a86b377SEli Cohen goto err_rqt; 16711a86b377SEli Cohen } 16721a86b377SEli Cohen 16731a86b377SEli Cohen err = create_tir(ndev); 16741a86b377SEli Cohen if (err) { 16751a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "create_tir\n"); 16761a86b377SEli Cohen goto err_tir; 16771a86b377SEli Cohen } 16781a86b377SEli Cohen 16791a86b377SEli Cohen err = add_fwd_to_tir(ndev); 16801a86b377SEli Cohen if (err) { 16811a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "add_fwd_to_tir\n"); 16821a86b377SEli Cohen goto err_fwd; 16831a86b377SEli Cohen } 16841a86b377SEli Cohen ndev->setup = true; 16851a86b377SEli Cohen mutex_unlock(&ndev->reslock); 16861a86b377SEli Cohen 16871a86b377SEli Cohen return 0; 16881a86b377SEli Cohen 16891a86b377SEli Cohen err_fwd: 16901a86b377SEli Cohen destroy_tir(ndev); 16911a86b377SEli Cohen err_tir: 16921a86b377SEli Cohen destroy_rqt(ndev); 16931a86b377SEli Cohen err_rqt: 16941a86b377SEli Cohen teardown_virtqueues(ndev); 16951a86b377SEli Cohen out: 16961a86b377SEli Cohen mutex_unlock(&ndev->reslock); 16971a86b377SEli Cohen return err; 16981a86b377SEli Cohen } 16991a86b377SEli Cohen 17001a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev) 17011a86b377SEli Cohen { 17021a86b377SEli Cohen mutex_lock(&ndev->reslock); 17031a86b377SEli Cohen if (!ndev->setup) 17041a86b377SEli Cohen goto out; 17051a86b377SEli Cohen 17061a86b377SEli Cohen remove_fwd_to_tir(ndev); 17071a86b377SEli Cohen destroy_tir(ndev); 17081a86b377SEli Cohen destroy_rqt(ndev); 17091a86b377SEli Cohen teardown_virtqueues(ndev); 17101a86b377SEli Cohen ndev->setup = false; 17111a86b377SEli Cohen out: 17121a86b377SEli Cohen mutex_unlock(&ndev->reslock); 17131a86b377SEli Cohen } 17141a86b377SEli Cohen 17151a86b377SEli Cohen static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) 17161a86b377SEli Cohen { 17171a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17181a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17191a86b377SEli Cohen int err; 17201a86b377SEli Cohen 17211a86b377SEli Cohen print_status(mvdev, status, true); 17221a86b377SEli Cohen if (!status) { 17231a86b377SEli Cohen mlx5_vdpa_info(mvdev, "performing device reset\n"); 17241a86b377SEli Cohen teardown_driver(ndev); 17251a86b377SEli Cohen mlx5_vdpa_destroy_mr(&ndev->mvdev); 17261a86b377SEli Cohen ndev->mvdev.status = 0; 17271a86b377SEli Cohen ndev->mvdev.mlx_features = 0; 17281a86b377SEli Cohen ++mvdev->generation; 17291a86b377SEli Cohen return; 17301a86b377SEli Cohen } 17311a86b377SEli Cohen 17321a86b377SEli Cohen if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { 17331a86b377SEli Cohen if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 17341a86b377SEli Cohen err = setup_driver(ndev); 17351a86b377SEli Cohen if (err) { 17361a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); 17371a86b377SEli Cohen goto err_setup; 17381a86b377SEli Cohen } 17391a86b377SEli Cohen } else { 17401a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); 17411a86b377SEli Cohen return; 17421a86b377SEli Cohen } 17431a86b377SEli Cohen } 17441a86b377SEli Cohen 17451a86b377SEli Cohen ndev->mvdev.status = status; 17461a86b377SEli Cohen return; 17471a86b377SEli Cohen 17481a86b377SEli Cohen err_setup: 17491a86b377SEli Cohen mlx5_vdpa_destroy_mr(&ndev->mvdev); 17501a86b377SEli Cohen ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; 17511a86b377SEli Cohen } 17521a86b377SEli Cohen 17531a86b377SEli Cohen static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, 17541a86b377SEli Cohen unsigned int len) 17551a86b377SEli Cohen { 17561a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17571a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17581a86b377SEli Cohen 17591a86b377SEli Cohen if (offset + len < sizeof(struct virtio_net_config)) 1760*2874211fSDan Carpenter memcpy(buf, (u8 *)&ndev->config + offset, len); 17611a86b377SEli Cohen } 17621a86b377SEli Cohen 17631a86b377SEli Cohen static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf, 17641a86b377SEli Cohen unsigned int len) 17651a86b377SEli Cohen { 17661a86b377SEli Cohen /* not supported */ 17671a86b377SEli Cohen } 17681a86b377SEli Cohen 17691a86b377SEli Cohen static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) 17701a86b377SEli Cohen { 17711a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17721a86b377SEli Cohen 17731a86b377SEli Cohen return mvdev->generation; 17741a86b377SEli Cohen } 17751a86b377SEli Cohen 17761a86b377SEli Cohen static int mlx5_vdpa_set_map(struct vdpa_device *vdev, struct vhost_iotlb *iotlb) 17771a86b377SEli Cohen { 17781a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17791a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17801a86b377SEli Cohen bool change_map; 17811a86b377SEli Cohen int err; 17821a86b377SEli Cohen 17831a86b377SEli Cohen err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map); 17841a86b377SEli Cohen if (err) { 17851a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); 17861a86b377SEli Cohen return err; 17871a86b377SEli Cohen } 17881a86b377SEli Cohen 17891a86b377SEli Cohen if (change_map) 17901a86b377SEli Cohen return mlx5_vdpa_change_map(ndev, iotlb); 17911a86b377SEli Cohen 17921a86b377SEli Cohen return 0; 17931a86b377SEli Cohen } 17941a86b377SEli Cohen 17951a86b377SEli Cohen static void mlx5_vdpa_free(struct vdpa_device *vdev) 17961a86b377SEli Cohen { 17971a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17981a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 17991a86b377SEli Cohen 18001a86b377SEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 18011a86b377SEli Cohen 18021a86b377SEli Cohen free_resources(ndev); 18031a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 18041a86b377SEli Cohen mutex_destroy(&ndev->reslock); 18051a86b377SEli Cohen } 18061a86b377SEli Cohen 18071a86b377SEli Cohen static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx) 18081a86b377SEli Cohen { 18091a86b377SEli Cohen struct vdpa_notification_area ret = {}; 18101a86b377SEli Cohen 18111a86b377SEli Cohen return ret; 18121a86b377SEli Cohen } 18131a86b377SEli Cohen 18141a86b377SEli Cohen static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) 18151a86b377SEli Cohen { 18161a86b377SEli Cohen return -EOPNOTSUPP; 18171a86b377SEli Cohen } 18181a86b377SEli Cohen 18191a86b377SEli Cohen static const struct vdpa_config_ops mlx5_vdpa_ops = { 18201a86b377SEli Cohen .set_vq_address = mlx5_vdpa_set_vq_address, 18211a86b377SEli Cohen .set_vq_num = mlx5_vdpa_set_vq_num, 18221a86b377SEli Cohen .kick_vq = mlx5_vdpa_kick_vq, 18231a86b377SEli Cohen .set_vq_cb = mlx5_vdpa_set_vq_cb, 18241a86b377SEli Cohen .set_vq_ready = mlx5_vdpa_set_vq_ready, 18251a86b377SEli Cohen .get_vq_ready = mlx5_vdpa_get_vq_ready, 18261a86b377SEli Cohen .set_vq_state = mlx5_vdpa_set_vq_state, 18271a86b377SEli Cohen .get_vq_state = mlx5_vdpa_get_vq_state, 18281a86b377SEli Cohen .get_vq_notification = mlx5_get_vq_notification, 18291a86b377SEli Cohen .get_vq_irq = mlx5_get_vq_irq, 18301a86b377SEli Cohen .get_vq_align = mlx5_vdpa_get_vq_align, 18311a86b377SEli Cohen .get_features = mlx5_vdpa_get_features, 18321a86b377SEli Cohen .set_features = mlx5_vdpa_set_features, 18331a86b377SEli Cohen .set_config_cb = mlx5_vdpa_set_config_cb, 18341a86b377SEli Cohen .get_vq_num_max = mlx5_vdpa_get_vq_num_max, 18351a86b377SEli Cohen .get_device_id = mlx5_vdpa_get_device_id, 18361a86b377SEli Cohen .get_vendor_id = mlx5_vdpa_get_vendor_id, 18371a86b377SEli Cohen .get_status = mlx5_vdpa_get_status, 18381a86b377SEli Cohen .set_status = mlx5_vdpa_set_status, 18391a86b377SEli Cohen .get_config = mlx5_vdpa_get_config, 18401a86b377SEli Cohen .set_config = mlx5_vdpa_set_config, 18411a86b377SEli Cohen .get_generation = mlx5_vdpa_get_generation, 18421a86b377SEli Cohen .set_map = mlx5_vdpa_set_map, 18431a86b377SEli Cohen .free = mlx5_vdpa_free, 18441a86b377SEli Cohen }; 18451a86b377SEli Cohen 18461a86b377SEli Cohen static int alloc_resources(struct mlx5_vdpa_net *ndev) 18471a86b377SEli Cohen { 18481a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 18491a86b377SEli Cohen int err; 18501a86b377SEli Cohen 18511a86b377SEli Cohen if (res->valid) { 18521a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); 18531a86b377SEli Cohen return -EEXIST; 18541a86b377SEli Cohen } 18551a86b377SEli Cohen 18561a86b377SEli Cohen err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); 18571a86b377SEli Cohen if (err) 18581a86b377SEli Cohen return err; 18591a86b377SEli Cohen 18601a86b377SEli Cohen err = create_tis(ndev); 18611a86b377SEli Cohen if (err) 18621a86b377SEli Cohen goto err_tis; 18631a86b377SEli Cohen 18641a86b377SEli Cohen res->valid = true; 18651a86b377SEli Cohen 18661a86b377SEli Cohen return 0; 18671a86b377SEli Cohen 18681a86b377SEli Cohen err_tis: 18691a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 18701a86b377SEli Cohen return err; 18711a86b377SEli Cohen } 18721a86b377SEli Cohen 18731a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev) 18741a86b377SEli Cohen { 18751a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 18761a86b377SEli Cohen 18771a86b377SEli Cohen if (!res->valid) 18781a86b377SEli Cohen return; 18791a86b377SEli Cohen 18801a86b377SEli Cohen destroy_tis(ndev); 18811a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 18821a86b377SEli Cohen res->valid = false; 18831a86b377SEli Cohen } 18841a86b377SEli Cohen 18851a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev) 18861a86b377SEli Cohen { 18871a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 18881a86b377SEli Cohen int i; 18891a86b377SEli Cohen 18901a86b377SEli Cohen for (i = 0; i < 2 * mlx5_vdpa_max_qps(ndev->mvdev.max_vqs); ++i) { 18911a86b377SEli Cohen mvq = &ndev->vqs[i]; 18921a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 18931a86b377SEli Cohen mvq->index = i; 18941a86b377SEli Cohen mvq->ndev = ndev; 18951a86b377SEli Cohen mvq->fwqp.fw = true; 18961a86b377SEli Cohen } 18971a86b377SEli Cohen for (; i < ndev->mvdev.max_vqs; i++) { 18981a86b377SEli Cohen mvq = &ndev->vqs[i]; 18991a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 19001a86b377SEli Cohen mvq->index = i; 19011a86b377SEli Cohen mvq->ndev = ndev; 19021a86b377SEli Cohen } 19031a86b377SEli Cohen } 19041a86b377SEli Cohen 19051a86b377SEli Cohen void *mlx5_vdpa_add_dev(struct mlx5_core_dev *mdev) 19061a86b377SEli Cohen { 19071a86b377SEli Cohen struct virtio_net_config *config; 19081a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev; 19091a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 19101a86b377SEli Cohen u32 max_vqs; 19111a86b377SEli Cohen int err; 19121a86b377SEli Cohen 19131a86b377SEli Cohen /* we save one virtqueue for control virtqueue should we require it */ 19141a86b377SEli Cohen max_vqs = MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues); 19151a86b377SEli Cohen max_vqs = min_t(u32, max_vqs, MLX5_MAX_SUPPORTED_VQS); 19161a86b377SEli Cohen 19171a86b377SEli Cohen ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, 19181a86b377SEli Cohen 2 * mlx5_vdpa_max_qps(max_vqs)); 19191a86b377SEli Cohen if (IS_ERR(ndev)) 19201a86b377SEli Cohen return ndev; 19211a86b377SEli Cohen 19221a86b377SEli Cohen ndev->mvdev.max_vqs = max_vqs; 19231a86b377SEli Cohen mvdev = &ndev->mvdev; 19241a86b377SEli Cohen mvdev->mdev = mdev; 19251a86b377SEli Cohen init_mvqs(ndev); 19261a86b377SEli Cohen mutex_init(&ndev->reslock); 19271a86b377SEli Cohen config = &ndev->config; 19281a86b377SEli Cohen err = mlx5_query_nic_vport_mtu(mdev, &config->mtu); 19291a86b377SEli Cohen if (err) 19301a86b377SEli Cohen goto err_mtu; 19311a86b377SEli Cohen 19321a86b377SEli Cohen err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac); 19331a86b377SEli Cohen if (err) 19341a86b377SEli Cohen goto err_mtu; 19351a86b377SEli Cohen 19361a86b377SEli Cohen mvdev->vdev.dma_dev = mdev->device; 19371a86b377SEli Cohen err = mlx5_vdpa_alloc_resources(&ndev->mvdev); 19381a86b377SEli Cohen if (err) 19391a86b377SEli Cohen goto err_mtu; 19401a86b377SEli Cohen 19411a86b377SEli Cohen err = alloc_resources(ndev); 19421a86b377SEli Cohen if (err) 19431a86b377SEli Cohen goto err_res; 19441a86b377SEli Cohen 19451a86b377SEli Cohen err = vdpa_register_device(&mvdev->vdev); 19461a86b377SEli Cohen if (err) 19471a86b377SEli Cohen goto err_reg; 19481a86b377SEli Cohen 19491a86b377SEli Cohen return ndev; 19501a86b377SEli Cohen 19511a86b377SEli Cohen err_reg: 19521a86b377SEli Cohen free_resources(ndev); 19531a86b377SEli Cohen err_res: 19541a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 19551a86b377SEli Cohen err_mtu: 19561a86b377SEli Cohen mutex_destroy(&ndev->reslock); 19571a86b377SEli Cohen put_device(&mvdev->vdev.dev); 19581a86b377SEli Cohen return ERR_PTR(err); 19591a86b377SEli Cohen } 19601a86b377SEli Cohen 19611a86b377SEli Cohen void mlx5_vdpa_remove_dev(struct mlx5_vdpa_dev *mvdev) 19621a86b377SEli Cohen { 19631a86b377SEli Cohen vdpa_unregister_device(&mvdev->vdev); 19641a86b377SEli Cohen } 1965