11a86b377SEli Cohen // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 21a86b377SEli Cohen /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 31a86b377SEli Cohen 474c9729dSLeon Romanovsky #include <linux/module.h> 51a86b377SEli Cohen #include <linux/vdpa.h> 674c9729dSLeon Romanovsky #include <linux/vringh.h> 774c9729dSLeon Romanovsky #include <uapi/linux/virtio_net.h> 81a86b377SEli Cohen #include <uapi/linux/virtio_ids.h> 91a86b377SEli Cohen #include <linux/virtio_config.h> 1074c9729dSLeon Romanovsky #include <linux/auxiliary_bus.h> 1174c9729dSLeon Romanovsky #include <linux/mlx5/cq.h> 121a86b377SEli Cohen #include <linux/mlx5/qp.h> 131a86b377SEli Cohen #include <linux/mlx5/device.h> 1474c9729dSLeon Romanovsky #include <linux/mlx5/driver.h> 151a86b377SEli Cohen #include <linux/mlx5/vport.h> 161a86b377SEli Cohen #include <linux/mlx5/fs.h> 170aae392bSLeon Romanovsky #include <linux/mlx5/mlx5_ifc_vdpa.h> 187c9f131fSEli Cohen #include <linux/mlx5/mpfs.h> 191a86b377SEli Cohen #include "mlx5_vdpa.h" 201a86b377SEli Cohen 2174c9729dSLeon Romanovsky MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 2274c9729dSLeon Romanovsky MODULE_DESCRIPTION("Mellanox VDPA driver"); 2374c9729dSLeon Romanovsky MODULE_LICENSE("Dual BSD/GPL"); 2474c9729dSLeon Romanovsky 2574c9729dSLeon Romanovsky #define to_mlx5_vdpa_ndev(__mvdev) \ 2674c9729dSLeon Romanovsky container_of(__mvdev, struct mlx5_vdpa_net, mvdev) 271a86b377SEli Cohen #define to_mvdev(__vdev) container_of((__vdev), struct mlx5_vdpa_dev, vdev) 281a86b377SEli Cohen 291a86b377SEli Cohen #define VALID_FEATURES_MASK \ 30cbb52359SNathan Chancellor (BIT_ULL(VIRTIO_NET_F_CSUM) | BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | \ 31cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_MAC) | \ 32cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | \ 33cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | \ 34cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | BIT_ULL(VIRTIO_NET_F_HOST_ECN) | BIT_ULL(VIRTIO_NET_F_HOST_UFO) | \ 35cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | BIT_ULL(VIRTIO_NET_F_STATUS) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | \ 36cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_RX) | BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | \ 37cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE) | \ 38cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | \ 39cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_RSS) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY) | \ 40cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX) | BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY) | \ 41cbb52359SNathan Chancellor BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | \ 42cbb52359SNathan Chancellor BIT_ULL(VIRTIO_F_RING_PACKED) | BIT_ULL(VIRTIO_F_ORDER_PLATFORM) | BIT_ULL(VIRTIO_F_SR_IOV)) 431a86b377SEli Cohen 441a86b377SEli Cohen #define VALID_STATUS_MASK \ 451a86b377SEli Cohen (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \ 461a86b377SEli Cohen VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED) 471a86b377SEli Cohen 48e4fc6650SEli Cohen #define MLX5_FEATURE(_mvdev, _feature) (!!((_mvdev)->actual_features & BIT_ULL(_feature))) 49e4fc6650SEli Cohen 501a86b377SEli Cohen struct mlx5_vdpa_net_resources { 511a86b377SEli Cohen u32 tisn; 521a86b377SEli Cohen u32 tdn; 531a86b377SEli Cohen u32 tirn; 541a86b377SEli Cohen u32 rqtn; 551a86b377SEli Cohen bool valid; 561a86b377SEli Cohen }; 571a86b377SEli Cohen 581a86b377SEli Cohen struct mlx5_vdpa_cq_buf { 591a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 601a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 611a86b377SEli Cohen int cqe_size; 621a86b377SEli Cohen int nent; 631a86b377SEli Cohen }; 641a86b377SEli Cohen 651a86b377SEli Cohen struct mlx5_vdpa_cq { 661a86b377SEli Cohen struct mlx5_core_cq mcq; 671a86b377SEli Cohen struct mlx5_vdpa_cq_buf buf; 681a86b377SEli Cohen struct mlx5_db db; 691a86b377SEli Cohen int cqe; 701a86b377SEli Cohen }; 711a86b377SEli Cohen 721a86b377SEli Cohen struct mlx5_vdpa_umem { 731a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 741a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 751a86b377SEli Cohen int size; 761a86b377SEli Cohen u32 id; 771a86b377SEli Cohen }; 781a86b377SEli Cohen 791a86b377SEli Cohen struct mlx5_vdpa_qp { 801a86b377SEli Cohen struct mlx5_core_qp mqp; 811a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 821a86b377SEli Cohen struct mlx5_db db; 831a86b377SEli Cohen u16 head; 841a86b377SEli Cohen bool fw; 851a86b377SEli Cohen }; 861a86b377SEli Cohen 871a86b377SEli Cohen struct mlx5_vq_restore_info { 881a86b377SEli Cohen u32 num_ent; 891a86b377SEli Cohen u64 desc_addr; 901a86b377SEli Cohen u64 device_addr; 911a86b377SEli Cohen u64 driver_addr; 921a86b377SEli Cohen u16 avail_index; 93b35ccebeSEli Cohen u16 used_index; 941a86b377SEli Cohen bool ready; 951a86b377SEli Cohen bool restore; 961a86b377SEli Cohen }; 971a86b377SEli Cohen 981a86b377SEli Cohen struct mlx5_vdpa_virtqueue { 991a86b377SEli Cohen bool ready; 1001a86b377SEli Cohen u64 desc_addr; 1011a86b377SEli Cohen u64 device_addr; 1021a86b377SEli Cohen u64 driver_addr; 1031a86b377SEli Cohen u32 num_ent; 1041a86b377SEli Cohen 1051a86b377SEli Cohen /* Resources for implementing the notification channel from the device 1061a86b377SEli Cohen * to the driver. fwqp is the firmware end of an RC connection; the 1071a86b377SEli Cohen * other end is vqqp used by the driver. cq is is where completions are 1081a86b377SEli Cohen * reported. 1091a86b377SEli Cohen */ 1101a86b377SEli Cohen struct mlx5_vdpa_cq cq; 1111a86b377SEli Cohen struct mlx5_vdpa_qp fwqp; 1121a86b377SEli Cohen struct mlx5_vdpa_qp vqqp; 1131a86b377SEli Cohen 1141a86b377SEli Cohen /* umem resources are required for the virtqueue operation. They're use 1151a86b377SEli Cohen * is internal and they must be provided by the driver. 1161a86b377SEli Cohen */ 1171a86b377SEli Cohen struct mlx5_vdpa_umem umem1; 1181a86b377SEli Cohen struct mlx5_vdpa_umem umem2; 1191a86b377SEli Cohen struct mlx5_vdpa_umem umem3; 1201a86b377SEli Cohen 1211a86b377SEli Cohen bool initialized; 1221a86b377SEli Cohen int index; 1231a86b377SEli Cohen u32 virtq_id; 1241a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 1251a86b377SEli Cohen u16 avail_idx; 126b35ccebeSEli Cohen u16 used_idx; 1271a86b377SEli Cohen int fw_state; 1281a86b377SEli Cohen 1291a86b377SEli Cohen /* keep last in the struct */ 1301a86b377SEli Cohen struct mlx5_vq_restore_info ri; 1311a86b377SEli Cohen }; 1321a86b377SEli Cohen 1331a86b377SEli Cohen /* We will remove this limitation once mlx5_vdpa_alloc_resources() 1341a86b377SEli Cohen * provides for driver space allocation 1351a86b377SEli Cohen */ 1361a86b377SEli Cohen #define MLX5_MAX_SUPPORTED_VQS 16 1371a86b377SEli Cohen 138e4fc6650SEli Cohen static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx) 139e4fc6650SEli Cohen { 140e4fc6650SEli Cohen if (unlikely(idx > mvdev->max_idx)) 141e4fc6650SEli Cohen return false; 142e4fc6650SEli Cohen 143e4fc6650SEli Cohen return true; 144e4fc6650SEli Cohen } 145e4fc6650SEli Cohen 1461a86b377SEli Cohen struct mlx5_vdpa_net { 1471a86b377SEli Cohen struct mlx5_vdpa_dev mvdev; 1481a86b377SEli Cohen struct mlx5_vdpa_net_resources res; 1491a86b377SEli Cohen struct virtio_net_config config; 1501a86b377SEli Cohen struct mlx5_vdpa_virtqueue vqs[MLX5_MAX_SUPPORTED_VQS]; 151db296d25SEli Cohen struct vdpa_callback event_cbs[MLX5_MAX_SUPPORTED_VQS + 1]; 1521a86b377SEli Cohen 1531a86b377SEli Cohen /* Serialize vq resources creation and destruction. This is required 1541a86b377SEli Cohen * since memory map might change and we need to destroy and create 1551a86b377SEli Cohen * resources while driver in operational. 1561a86b377SEli Cohen */ 1571a86b377SEli Cohen struct mutex reslock; 1581a86b377SEli Cohen struct mlx5_flow_table *rxft; 1591a86b377SEli Cohen struct mlx5_fc *rx_counter; 1601a86b377SEli Cohen struct mlx5_flow_handle *rx_rule; 1611a86b377SEli Cohen bool setup; 1625262912eSEli Cohen u32 cur_num_vqs; 1631a86b377SEli Cohen }; 1641a86b377SEli Cohen 1651a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev); 1661a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev); 167ae0428deSEli Cohen static int setup_driver(struct mlx5_vdpa_dev *mvdev); 1681a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev); 1691a86b377SEli Cohen 1701a86b377SEli Cohen static bool mlx5_vdpa_debug; 1711a86b377SEli Cohen 1725262912eSEli Cohen #define MLX5_CVQ_MAX_ENT 16 1735262912eSEli Cohen 1741a86b377SEli Cohen #define MLX5_LOG_VIO_FLAG(_feature) \ 1751a86b377SEli Cohen do { \ 176cbb52359SNathan Chancellor if (features & BIT_ULL(_feature)) \ 1771a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_feature); \ 1781a86b377SEli Cohen } while (0) 1791a86b377SEli Cohen 1801a86b377SEli Cohen #define MLX5_LOG_VIO_STAT(_status) \ 1811a86b377SEli Cohen do { \ 1821a86b377SEli Cohen if (status & (_status)) \ 1831a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_status); \ 1841a86b377SEli Cohen } while (0) 1851a86b377SEli Cohen 18652893733SEli Cohen /* TODO: cross-endian support */ 18752893733SEli Cohen static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev) 18852893733SEli Cohen { 18952893733SEli Cohen return virtio_legacy_is_little_endian() || 19052893733SEli Cohen (mvdev->actual_features & BIT_ULL(VIRTIO_F_VERSION_1)); 19152893733SEli Cohen } 19252893733SEli Cohen 19352893733SEli Cohen static u16 mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev *mvdev, __virtio16 val) 19452893733SEli Cohen { 19552893733SEli Cohen return __virtio16_to_cpu(mlx5_vdpa_is_little_endian(mvdev), val); 19652893733SEli Cohen } 19752893733SEli Cohen 19852893733SEli Cohen static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val) 19952893733SEli Cohen { 20052893733SEli Cohen return __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), val); 20152893733SEli Cohen } 20252893733SEli Cohen 20374c9729dSLeon Romanovsky static inline u32 mlx5_vdpa_max_qps(int max_vqs) 20474c9729dSLeon Romanovsky { 20574c9729dSLeon Romanovsky return max_vqs / 2; 20674c9729dSLeon Romanovsky } 20774c9729dSLeon Romanovsky 2085262912eSEli Cohen static u16 ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev) 2095262912eSEli Cohen { 21052893733SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) 21152893733SEli Cohen return 2; 21252893733SEli Cohen 2135262912eSEli Cohen return 2 * mlx5_vdpa_max_qps(mvdev->max_vqs); 2145262912eSEli Cohen } 2155262912eSEli Cohen 2165262912eSEli Cohen static bool is_ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev, u16 idx) 2175262912eSEli Cohen { 2185262912eSEli Cohen return idx == ctrl_vq_idx(mvdev); 2195262912eSEli Cohen } 2205262912eSEli Cohen 2211a86b377SEli Cohen static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set) 2221a86b377SEli Cohen { 2231a86b377SEli Cohen if (status & ~VALID_STATUS_MASK) 2241a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n", 2251a86b377SEli Cohen status & ~VALID_STATUS_MASK); 2261a86b377SEli Cohen 2271a86b377SEli Cohen if (!mlx5_vdpa_debug) 2281a86b377SEli Cohen return; 2291a86b377SEli Cohen 2301a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get"); 2311a86b377SEli Cohen if (set && !status) { 2321a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver resets the device\n"); 2331a86b377SEli Cohen return; 2341a86b377SEli Cohen } 2351a86b377SEli Cohen 2361a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE); 2371a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER); 2381a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK); 2391a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK); 2401a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET); 2411a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED); 2421a86b377SEli Cohen } 2431a86b377SEli Cohen 2441a86b377SEli Cohen static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set) 2451a86b377SEli Cohen { 2461a86b377SEli Cohen if (features & ~VALID_FEATURES_MASK) 2471a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n", 2481a86b377SEli Cohen features & ~VALID_FEATURES_MASK); 2491a86b377SEli Cohen 2501a86b377SEli Cohen if (!mlx5_vdpa_debug) 2511a86b377SEli Cohen return; 2521a86b377SEli Cohen 2531a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads"); 2541a86b377SEli Cohen if (!features) 2551a86b377SEli Cohen mlx5_vdpa_info(mvdev, "all feature bits are cleared\n"); 2561a86b377SEli Cohen 2571a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM); 2581a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM); 2591a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 2601a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU); 2611a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC); 2621a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4); 2631a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6); 2641a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN); 2651a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO); 2661a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4); 2671a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6); 2681a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN); 2691a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO); 2701a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF); 2711a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS); 2721a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ); 2731a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX); 2741a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN); 2751a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA); 2761a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE); 2771a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ); 2781a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR); 2791a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT); 2801a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS); 2811a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT); 2821a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY); 2831a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX); 2841a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY); 2851a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT); 2861a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1); 2871a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM); 2881a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED); 2891a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM); 2901a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV); 2911a86b377SEli Cohen } 2921a86b377SEli Cohen 2931a86b377SEli Cohen static int create_tis(struct mlx5_vdpa_net *ndev) 2941a86b377SEli Cohen { 2951a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; 2961a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 2971a86b377SEli Cohen void *tisc; 2981a86b377SEli Cohen int err; 2991a86b377SEli Cohen 3001a86b377SEli Cohen tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3011a86b377SEli Cohen MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); 3021a86b377SEli Cohen err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); 3031a86b377SEli Cohen if (err) 3041a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err); 3051a86b377SEli Cohen 3061a86b377SEli Cohen return err; 3071a86b377SEli Cohen } 3081a86b377SEli Cohen 3091a86b377SEli Cohen static void destroy_tis(struct mlx5_vdpa_net *ndev) 3101a86b377SEli Cohen { 3111a86b377SEli Cohen mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); 3121a86b377SEli Cohen } 3131a86b377SEli Cohen 3141a86b377SEli Cohen #define MLX5_VDPA_CQE_SIZE 64 3151a86b377SEli Cohen #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE) 3161a86b377SEli Cohen 3171a86b377SEli Cohen static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) 3181a86b377SEli Cohen { 3191a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &buf->frag_buf; 3201a86b377SEli Cohen u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE; 3211a86b377SEli Cohen u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE; 3221a86b377SEli Cohen int err; 3231a86b377SEli Cohen 3241a86b377SEli Cohen err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, 3251a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3261a86b377SEli Cohen if (err) 3271a86b377SEli Cohen return err; 3281a86b377SEli Cohen 3291a86b377SEli Cohen mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc); 3301a86b377SEli Cohen 3311a86b377SEli Cohen buf->cqe_size = MLX5_VDPA_CQE_SIZE; 3321a86b377SEli Cohen buf->nent = nent; 3331a86b377SEli Cohen 3341a86b377SEli Cohen return 0; 3351a86b377SEli Cohen } 3361a86b377SEli Cohen 3371a86b377SEli Cohen static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) 3381a86b377SEli Cohen { 3391a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &umem->frag_buf; 3401a86b377SEli Cohen 3411a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, 3421a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3431a86b377SEli Cohen } 3441a86b377SEli Cohen 3451a86b377SEli Cohen static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) 3461a86b377SEli Cohen { 3471a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); 3481a86b377SEli Cohen } 3491a86b377SEli Cohen 3501a86b377SEli Cohen static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n) 3511a86b377SEli Cohen { 3521a86b377SEli Cohen return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n); 3531a86b377SEli Cohen } 3541a86b377SEli Cohen 3551a86b377SEli Cohen static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf) 3561a86b377SEli Cohen { 3571a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 3581a86b377SEli Cohen void *cqe; 3591a86b377SEli Cohen int i; 3601a86b377SEli Cohen 3611a86b377SEli Cohen for (i = 0; i < buf->nent; i++) { 3621a86b377SEli Cohen cqe = get_cqe(vcq, i); 3631a86b377SEli Cohen cqe64 = cqe; 3641a86b377SEli Cohen cqe64->op_own = MLX5_CQE_INVALID << 4; 3651a86b377SEli Cohen } 3661a86b377SEli Cohen } 3671a86b377SEli Cohen 3681a86b377SEli Cohen static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n) 3691a86b377SEli Cohen { 3701a86b377SEli Cohen struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1)); 3711a86b377SEli Cohen 3721a86b377SEli Cohen if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && 3731a86b377SEli Cohen !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe))) 3741a86b377SEli Cohen return cqe64; 3751a86b377SEli Cohen 3761a86b377SEli Cohen return NULL; 3771a86b377SEli Cohen } 3781a86b377SEli Cohen 3791a86b377SEli Cohen static void rx_post(struct mlx5_vdpa_qp *vqp, int n) 3801a86b377SEli Cohen { 3811a86b377SEli Cohen vqp->head += n; 3821a86b377SEli Cohen vqp->db.db[0] = cpu_to_be32(vqp->head); 3831a86b377SEli Cohen } 3841a86b377SEli Cohen 3851a86b377SEli Cohen static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, 3861a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq, u32 num_ent) 3871a86b377SEli Cohen { 3881a86b377SEli Cohen struct mlx5_vdpa_qp *vqp; 3891a86b377SEli Cohen __be64 *pas; 3901a86b377SEli Cohen void *qpc; 3911a86b377SEli Cohen 3921a86b377SEli Cohen vqp = fw ? &mvq->fwqp : &mvq->vqqp; 3931a86b377SEli Cohen MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); 3941a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 3951a86b377SEli Cohen if (vqp->fw) { 3961a86b377SEli Cohen /* Firmware QP is allocated by the driver for the firmware's 3971a86b377SEli Cohen * use so we can skip part of the params as they will be chosen by firmware 3981a86b377SEli Cohen */ 3991a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4001a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ); 4011a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 4021a86b377SEli Cohen return; 4031a86b377SEli Cohen } 4041a86b377SEli Cohen 4051a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4061a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4071a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4081a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4091a86b377SEli Cohen MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); 4101a86b377SEli Cohen MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 4111a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 4121a86b377SEli Cohen MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn); 4131a86b377SEli Cohen MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent)); 4141a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 4151a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas); 4161a86b377SEli Cohen mlx5_fill_page_frag_array(&vqp->frag_buf, pas); 4171a86b377SEli Cohen } 4181a86b377SEli Cohen 4191a86b377SEli Cohen static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) 4201a86b377SEli Cohen { 4211a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, 4221a86b377SEli Cohen num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf, 4231a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 4241a86b377SEli Cohen } 4251a86b377SEli Cohen 4261a86b377SEli Cohen static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 4271a86b377SEli Cohen { 4281a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); 4291a86b377SEli Cohen } 4301a86b377SEli Cohen 4311a86b377SEli Cohen static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 4321a86b377SEli Cohen struct mlx5_vdpa_qp *vqp) 4331a86b377SEli Cohen { 4341a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 4351a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_qp_in); 4361a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 4371a86b377SEli Cohen void *qpc; 4381a86b377SEli Cohen void *in; 4391a86b377SEli Cohen int err; 4401a86b377SEli Cohen 4411a86b377SEli Cohen if (!vqp->fw) { 4421a86b377SEli Cohen vqp = &mvq->vqqp; 4431a86b377SEli Cohen err = rq_buf_alloc(ndev, vqp, mvq->num_ent); 4441a86b377SEli Cohen if (err) 4451a86b377SEli Cohen return err; 4461a86b377SEli Cohen 4471a86b377SEli Cohen err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); 4481a86b377SEli Cohen if (err) 4491a86b377SEli Cohen goto err_db; 4501a86b377SEli Cohen inlen += vqp->frag_buf.npages * sizeof(__be64); 4511a86b377SEli Cohen } 4521a86b377SEli Cohen 4531a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 4541a86b377SEli Cohen if (!in) { 4551a86b377SEli Cohen err = -ENOMEM; 4561a86b377SEli Cohen goto err_kzalloc; 4571a86b377SEli Cohen } 4581a86b377SEli Cohen 4591a86b377SEli Cohen qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); 4601a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4611a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4621a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4631a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4641a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4651a86b377SEli Cohen if (!vqp->fw) 4661a86b377SEli Cohen MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma); 4671a86b377SEli Cohen MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 4681a86b377SEli Cohen err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 4691a86b377SEli Cohen kfree(in); 4701a86b377SEli Cohen if (err) 4711a86b377SEli Cohen goto err_kzalloc; 4721a86b377SEli Cohen 4731a86b377SEli Cohen vqp->mqp.uid = ndev->mvdev.res.uid; 4741a86b377SEli Cohen vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn); 4751a86b377SEli Cohen 4761a86b377SEli Cohen if (!vqp->fw) 4771a86b377SEli Cohen rx_post(vqp, mvq->num_ent); 4781a86b377SEli Cohen 4791a86b377SEli Cohen return 0; 4801a86b377SEli Cohen 4811a86b377SEli Cohen err_kzalloc: 4821a86b377SEli Cohen if (!vqp->fw) 4831a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 4841a86b377SEli Cohen err_db: 4851a86b377SEli Cohen if (!vqp->fw) 4861a86b377SEli Cohen rq_buf_free(ndev, vqp); 4871a86b377SEli Cohen 4881a86b377SEli Cohen return err; 4891a86b377SEli Cohen } 4901a86b377SEli Cohen 4911a86b377SEli Cohen static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 4921a86b377SEli Cohen { 4931a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 4941a86b377SEli Cohen 4951a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 4961a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn); 4971a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); 4981a86b377SEli Cohen if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) 4991a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); 5001a86b377SEli Cohen if (!vqp->fw) { 5011a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 5021a86b377SEli Cohen rq_buf_free(ndev, vqp); 5031a86b377SEli Cohen } 5041a86b377SEli Cohen } 5051a86b377SEli Cohen 5061a86b377SEli Cohen static void *next_cqe_sw(struct mlx5_vdpa_cq *cq) 5071a86b377SEli Cohen { 5081a86b377SEli Cohen return get_sw_cqe(cq, cq->mcq.cons_index); 5091a86b377SEli Cohen } 5101a86b377SEli Cohen 5111a86b377SEli Cohen static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq) 5121a86b377SEli Cohen { 5131a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 5141a86b377SEli Cohen 5151a86b377SEli Cohen cqe64 = next_cqe_sw(vcq); 5161a86b377SEli Cohen if (!cqe64) 5171a86b377SEli Cohen return -EAGAIN; 5181a86b377SEli Cohen 5191a86b377SEli Cohen vcq->mcq.cons_index++; 5201a86b377SEli Cohen return 0; 5211a86b377SEli Cohen } 5221a86b377SEli Cohen 5231a86b377SEli Cohen static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num) 5241a86b377SEli Cohen { 525db296d25SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 526db296d25SEli Cohen struct vdpa_callback *event_cb; 527db296d25SEli Cohen 528db296d25SEli Cohen event_cb = &ndev->event_cbs[mvq->index]; 5291a86b377SEli Cohen mlx5_cq_set_ci(&mvq->cq.mcq); 53083ef73b2SEli Cohen 53183ef73b2SEli Cohen /* make sure CQ cosumer update is visible to the hardware before updating 53283ef73b2SEli Cohen * RX doorbell record. 53383ef73b2SEli Cohen */ 53483ef73b2SEli Cohen dma_wmb(); 5351a86b377SEli Cohen rx_post(&mvq->vqqp, num); 536db296d25SEli Cohen if (event_cb->callback) 537db296d25SEli Cohen event_cb->callback(event_cb->private); 5381a86b377SEli Cohen } 5391a86b377SEli Cohen 5401a86b377SEli Cohen static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) 5411a86b377SEli Cohen { 5421a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq); 5431a86b377SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 5441a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5451a86b377SEli Cohen int num = 0; 5461a86b377SEli Cohen 5471a86b377SEli Cohen while (!mlx5_vdpa_poll_one(&mvq->cq)) { 5481a86b377SEli Cohen num++; 5491a86b377SEli Cohen if (num > mvq->num_ent / 2) { 5501a86b377SEli Cohen /* If completions keep coming while we poll, we want to 5511a86b377SEli Cohen * let the hardware know that we consumed them by 5521a86b377SEli Cohen * updating the doorbell record. We also let vdpa core 5531a86b377SEli Cohen * know about this so it passes it on the virtio driver 5541a86b377SEli Cohen * on the guest. 5551a86b377SEli Cohen */ 5561a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 5571a86b377SEli Cohen num = 0; 5581a86b377SEli Cohen } 5591a86b377SEli Cohen } 5601a86b377SEli Cohen 5611a86b377SEli Cohen if (num) 5621a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 5631a86b377SEli Cohen 5641a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 5651a86b377SEli Cohen } 5661a86b377SEli Cohen 5671a86b377SEli Cohen static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) 5681a86b377SEli Cohen { 5691a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 5701a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 5711a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5721a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 5731a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 5741a86b377SEli Cohen __be64 *pas; 5751a86b377SEli Cohen int inlen; 5761a86b377SEli Cohen void *cqc; 5771a86b377SEli Cohen void *in; 5781a86b377SEli Cohen int err; 5791a86b377SEli Cohen int eqn; 5801a86b377SEli Cohen 5811a86b377SEli Cohen err = mlx5_db_alloc(mdev, &vcq->db); 5821a86b377SEli Cohen if (err) 5831a86b377SEli Cohen return err; 5841a86b377SEli Cohen 5851a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 5861a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 5871a86b377SEli Cohen vcq->mcq.cqe_sz = 64; 5881a86b377SEli Cohen 5891a86b377SEli Cohen err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); 5901a86b377SEli Cohen if (err) 5911a86b377SEli Cohen goto err_db; 5921a86b377SEli Cohen 5931a86b377SEli Cohen cq_frag_buf_init(vcq, &vcq->buf); 5941a86b377SEli Cohen 5951a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 5961a86b377SEli Cohen MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages; 5971a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 5981a86b377SEli Cohen if (!in) { 5991a86b377SEli Cohen err = -ENOMEM; 6001a86b377SEli Cohen goto err_vzalloc; 6011a86b377SEli Cohen } 6021a86b377SEli Cohen 6031a86b377SEli Cohen MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); 6041a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 6051a86b377SEli Cohen mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas); 6061a86b377SEli Cohen 6071a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 6081a86b377SEli Cohen MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 6091a86b377SEli Cohen 6101a86b377SEli Cohen /* Use vector 0 by default. Consider adding code to choose least used 6111a86b377SEli Cohen * vector. 6121a86b377SEli Cohen */ 613563476aeSShay Drory err = mlx5_vector2eqn(mdev, 0, &eqn); 6141a86b377SEli Cohen if (err) 6151a86b377SEli Cohen goto err_vec; 6161a86b377SEli Cohen 6171a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 6181a86b377SEli Cohen MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent)); 6191a86b377SEli Cohen MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); 620616d5769STal Gilboa MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 6211a86b377SEli Cohen MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma); 6221a86b377SEli Cohen 6231a86b377SEli Cohen err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out)); 6241a86b377SEli Cohen if (err) 6251a86b377SEli Cohen goto err_vec; 6261a86b377SEli Cohen 6271a86b377SEli Cohen vcq->mcq.comp = mlx5_vdpa_cq_comp; 6281a86b377SEli Cohen vcq->cqe = num_ent; 6291a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 6301a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 6311a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 6321a86b377SEli Cohen kfree(in); 6331a86b377SEli Cohen return 0; 6341a86b377SEli Cohen 6351a86b377SEli Cohen err_vec: 6361a86b377SEli Cohen kfree(in); 6371a86b377SEli Cohen err_vzalloc: 6381a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 6391a86b377SEli Cohen err_db: 6401a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 6411a86b377SEli Cohen return err; 6421a86b377SEli Cohen } 6431a86b377SEli Cohen 6441a86b377SEli Cohen static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) 6451a86b377SEli Cohen { 6461a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 6471a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 6481a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 6491a86b377SEli Cohen 6501a86b377SEli Cohen if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) { 6511a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); 6521a86b377SEli Cohen return; 6531a86b377SEli Cohen } 6541a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 6551a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 6561a86b377SEli Cohen } 6571a86b377SEli Cohen 65871ab6a7cSEli Cohen static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, 6591a86b377SEli Cohen struct mlx5_vdpa_umem **umemp) 6601a86b377SEli Cohen { 6611a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 6621a86b377SEli Cohen int p_a; 6631a86b377SEli Cohen int p_b; 6641a86b377SEli Cohen 6651a86b377SEli Cohen switch (num) { 6661a86b377SEli Cohen case 1: 6671a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a); 6681a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b); 6691a86b377SEli Cohen *umemp = &mvq->umem1; 6701a86b377SEli Cohen break; 6711a86b377SEli Cohen case 2: 6721a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a); 6731a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b); 6741a86b377SEli Cohen *umemp = &mvq->umem2; 6751a86b377SEli Cohen break; 6761a86b377SEli Cohen case 3: 6771a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a); 6781a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b); 6791a86b377SEli Cohen *umemp = &mvq->umem3; 6801a86b377SEli Cohen break; 6811a86b377SEli Cohen } 68271ab6a7cSEli Cohen (*umemp)->size = p_a * mvq->num_ent + p_b; 6831a86b377SEli Cohen } 6841a86b377SEli Cohen 6851a86b377SEli Cohen static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) 6861a86b377SEli Cohen { 6871a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); 6881a86b377SEli Cohen } 6891a86b377SEli Cohen 6901a86b377SEli Cohen static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 6911a86b377SEli Cohen { 6921a86b377SEli Cohen int inlen; 6931a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {}; 6941a86b377SEli Cohen void *um; 6951a86b377SEli Cohen void *in; 6961a86b377SEli Cohen int err; 6971a86b377SEli Cohen __be64 *pas; 6981a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 6991a86b377SEli Cohen 70071ab6a7cSEli Cohen set_umem_size(ndev, mvq, num, &umem); 70171ab6a7cSEli Cohen err = umem_frag_buf_alloc(ndev, umem, umem->size); 7021a86b377SEli Cohen if (err) 7031a86b377SEli Cohen return err; 7041a86b377SEli Cohen 7051a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages; 7061a86b377SEli Cohen 7071a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 7081a86b377SEli Cohen if (!in) { 7091a86b377SEli Cohen err = -ENOMEM; 7101a86b377SEli Cohen goto err_in; 7111a86b377SEli Cohen } 7121a86b377SEli Cohen 7131a86b377SEli Cohen MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM); 7141a86b377SEli Cohen MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); 7151a86b377SEli Cohen um = MLX5_ADDR_OF(create_umem_in, in, umem); 7161a86b377SEli Cohen MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 7171a86b377SEli Cohen MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages); 7181a86b377SEli Cohen 7191a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]); 7201a86b377SEli Cohen mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW); 7211a86b377SEli Cohen 7221a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 7231a86b377SEli Cohen if (err) { 7241a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); 7251a86b377SEli Cohen goto err_cmd; 7261a86b377SEli Cohen } 7271a86b377SEli Cohen 7281a86b377SEli Cohen kfree(in); 7291a86b377SEli Cohen umem->id = MLX5_GET(create_umem_out, out, umem_id); 7301a86b377SEli Cohen 7311a86b377SEli Cohen return 0; 7321a86b377SEli Cohen 7331a86b377SEli Cohen err_cmd: 7341a86b377SEli Cohen kfree(in); 7351a86b377SEli Cohen err_in: 7361a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7371a86b377SEli Cohen return err; 7381a86b377SEli Cohen } 7391a86b377SEli Cohen 7401a86b377SEli Cohen static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 7411a86b377SEli Cohen { 7421a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {}; 7431a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {}; 7441a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 7451a86b377SEli Cohen 7461a86b377SEli Cohen switch (num) { 7471a86b377SEli Cohen case 1: 7481a86b377SEli Cohen umem = &mvq->umem1; 7491a86b377SEli Cohen break; 7501a86b377SEli Cohen case 2: 7511a86b377SEli Cohen umem = &mvq->umem2; 7521a86b377SEli Cohen break; 7531a86b377SEli Cohen case 3: 7541a86b377SEli Cohen umem = &mvq->umem3; 7551a86b377SEli Cohen break; 7561a86b377SEli Cohen } 7571a86b377SEli Cohen 7581a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM); 7591a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, umem_id, umem->id); 7601a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 7611a86b377SEli Cohen return; 7621a86b377SEli Cohen 7631a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7641a86b377SEli Cohen } 7651a86b377SEli Cohen 7661a86b377SEli Cohen static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7671a86b377SEli Cohen { 7681a86b377SEli Cohen int num; 7691a86b377SEli Cohen int err; 7701a86b377SEli Cohen 7711a86b377SEli Cohen for (num = 1; num <= 3; num++) { 7721a86b377SEli Cohen err = create_umem(ndev, mvq, num); 7731a86b377SEli Cohen if (err) 7741a86b377SEli Cohen goto err_umem; 7751a86b377SEli Cohen } 7761a86b377SEli Cohen return 0; 7771a86b377SEli Cohen 7781a86b377SEli Cohen err_umem: 7791a86b377SEli Cohen for (num--; num > 0; num--) 7801a86b377SEli Cohen umem_destroy(ndev, mvq, num); 7811a86b377SEli Cohen 7821a86b377SEli Cohen return err; 7831a86b377SEli Cohen } 7841a86b377SEli Cohen 7851a86b377SEli Cohen static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7861a86b377SEli Cohen { 7871a86b377SEli Cohen int num; 7881a86b377SEli Cohen 7891a86b377SEli Cohen for (num = 3; num > 0; num--) 7901a86b377SEli Cohen umem_destroy(ndev, mvq, num); 7911a86b377SEli Cohen } 7921a86b377SEli Cohen 7931a86b377SEli Cohen static int get_queue_type(struct mlx5_vdpa_net *ndev) 7941a86b377SEli Cohen { 7951a86b377SEli Cohen u32 type_mask; 7961a86b377SEli Cohen 7971a86b377SEli Cohen type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); 7981a86b377SEli Cohen 7991a86b377SEli Cohen /* prefer split queue */ 800879753c8SEli Cohen if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT) 8011a86b377SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT; 802879753c8SEli Cohen 803879753c8SEli Cohen WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED)); 804879753c8SEli Cohen 805879753c8SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED; 8061a86b377SEli Cohen } 8071a86b377SEli Cohen 8081a86b377SEli Cohen static bool vq_is_tx(u16 idx) 8091a86b377SEli Cohen { 8101a86b377SEli Cohen return idx % 2; 8111a86b377SEli Cohen } 8121a86b377SEli Cohen 8131a86b377SEli Cohen static u16 get_features_12_3(u64 features) 8141a86b377SEli Cohen { 815cbb52359SNathan Chancellor return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) | 816cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) | 817cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) | 818cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6); 8191a86b377SEli Cohen } 8201a86b377SEli Cohen 8211a86b377SEli Cohen static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 8221a86b377SEli Cohen { 8231a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); 8241a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {}; 8251a86b377SEli Cohen void *obj_context; 8261a86b377SEli Cohen void *cmd_hdr; 8271a86b377SEli Cohen void *vq_ctx; 8281a86b377SEli Cohen void *in; 8291a86b377SEli Cohen int err; 8301a86b377SEli Cohen 8311a86b377SEli Cohen err = umems_create(ndev, mvq); 8321a86b377SEli Cohen if (err) 8331a86b377SEli Cohen return err; 8341a86b377SEli Cohen 8351a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 8361a86b377SEli Cohen if (!in) { 8371a86b377SEli Cohen err = -ENOMEM; 8381a86b377SEli Cohen goto err_alloc; 8391a86b377SEli Cohen } 8401a86b377SEli Cohen 8411a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr); 8421a86b377SEli Cohen 8431a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 8441a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 8451a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 8461a86b377SEli Cohen 8471a86b377SEli Cohen obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context); 8481a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); 849b35ccebeSEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); 8501a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, 8511a86b377SEli Cohen get_features_12_3(ndev->mvdev.actual_features)); 8521a86b377SEli Cohen vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context); 8531a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); 8541a86b377SEli Cohen 8551a86b377SEli Cohen if (vq_is_tx(mvq->index)) 8561a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); 8571a86b377SEli Cohen 8581a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 8591a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); 8601a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 8611a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); 8621a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 8634b454a82SEli Cohen !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); 8641a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); 8651a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); 8661a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); 8671a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey.key); 8681a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); 8691a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); 8701a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id); 871e3011776SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem2.size); 8721a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id); 873e3011776SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem3.size); 8741a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); 8751a86b377SEli Cohen if (MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, eth_frame_offload_type)) 8761a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 1); 8771a86b377SEli Cohen 8781a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 8791a86b377SEli Cohen if (err) 8801a86b377SEli Cohen goto err_cmd; 8811a86b377SEli Cohen 8821a86b377SEli Cohen kfree(in); 8831a86b377SEli Cohen mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 8841a86b377SEli Cohen 8851a86b377SEli Cohen return 0; 8861a86b377SEli Cohen 8871a86b377SEli Cohen err_cmd: 8881a86b377SEli Cohen kfree(in); 8891a86b377SEli Cohen err_alloc: 8901a86b377SEli Cohen umems_destroy(ndev, mvq); 8911a86b377SEli Cohen return err; 8921a86b377SEli Cohen } 8931a86b377SEli Cohen 8941a86b377SEli Cohen static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 8951a86b377SEli Cohen { 8961a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {}; 8971a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {}; 8981a86b377SEli Cohen 8991a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode, 9001a86b377SEli Cohen MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 9011a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id); 9021a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); 9031a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type, 9041a86b377SEli Cohen MLX5_OBJ_TYPE_VIRTIO_NET_Q); 9051a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { 9061a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); 9071a86b377SEli Cohen return; 9081a86b377SEli Cohen } 9091a86b377SEli Cohen umems_destroy(ndev, mvq); 9101a86b377SEli Cohen } 9111a86b377SEli Cohen 9121a86b377SEli Cohen static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 9131a86b377SEli Cohen { 9141a86b377SEli Cohen return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn; 9151a86b377SEli Cohen } 9161a86b377SEli Cohen 9171a86b377SEli Cohen static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 9181a86b377SEli Cohen { 9191a86b377SEli Cohen return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn; 9201a86b377SEli Cohen } 9211a86b377SEli Cohen 9221a86b377SEli Cohen static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, 9231a86b377SEli Cohen int *outlen, u32 qpn, u32 rqpn) 9241a86b377SEli Cohen { 9251a86b377SEli Cohen void *qpc; 9261a86b377SEli Cohen void *pp; 9271a86b377SEli Cohen 9281a86b377SEli Cohen switch (cmd) { 9291a86b377SEli Cohen case MLX5_CMD_OP_2RST_QP: 9301a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in); 9311a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out); 9321a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9331a86b377SEli Cohen *out = kzalloc(*outlen, GFP_KERNEL); 934f31231bfSColin Ian King if (!*in || !*out) 9351a86b377SEli Cohen goto outerr; 9361a86b377SEli Cohen 9371a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, opcode, cmd); 9381a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); 9391a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, qpn, qpn); 9401a86b377SEli Cohen break; 9411a86b377SEli Cohen case MLX5_CMD_OP_RST2INIT_QP: 9421a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in); 9431a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out); 9441a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9451a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL); 946f31231bfSColin Ian King if (!*in || !*out) 9471a86b377SEli Cohen goto outerr; 9481a86b377SEli Cohen 9491a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, opcode, cmd); 9501a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); 9511a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, qpn, qpn); 9521a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9531a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9541a86b377SEli Cohen MLX5_SET(qpc, qpc, rwe, 1); 9551a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9561a86b377SEli Cohen MLX5_SET(ads, pp, vhca_port_num, 1); 9571a86b377SEli Cohen break; 9581a86b377SEli Cohen case MLX5_CMD_OP_INIT2RTR_QP: 9591a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in); 9601a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out); 9611a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9621a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL); 963f31231bfSColin Ian King if (!*in || !*out) 9641a86b377SEli Cohen goto outerr; 9651a86b377SEli Cohen 9661a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, opcode, cmd); 9671a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); 9681a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, qpn, qpn); 9691a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9701a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 9711a86b377SEli Cohen MLX5_SET(qpc, qpc, log_msg_max, 30); 9721a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9731a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9741a86b377SEli Cohen MLX5_SET(ads, pp, fl, 1); 9751a86b377SEli Cohen break; 9761a86b377SEli Cohen case MLX5_CMD_OP_RTR2RTS_QP: 9771a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in); 9781a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out); 9791a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9801a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL); 981f31231bfSColin Ian King if (!*in || !*out) 9821a86b377SEli Cohen goto outerr; 9831a86b377SEli Cohen 9841a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd); 9851a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); 9861a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn); 9871a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9881a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9891a86b377SEli Cohen MLX5_SET(ads, pp, ack_timeout, 14); 9901a86b377SEli Cohen MLX5_SET(qpc, qpc, retry_count, 7); 9911a86b377SEli Cohen MLX5_SET(qpc, qpc, rnr_retry, 7); 9921a86b377SEli Cohen break; 9931a86b377SEli Cohen default: 994f31231bfSColin Ian King goto outerr_nullify; 9951a86b377SEli Cohen } 9961a86b377SEli Cohen 9971a86b377SEli Cohen return; 9981a86b377SEli Cohen 9991a86b377SEli Cohen outerr: 10001a86b377SEli Cohen kfree(*in); 10011a86b377SEli Cohen kfree(*out); 1002f31231bfSColin Ian King outerr_nullify: 10031a86b377SEli Cohen *in = NULL; 10041a86b377SEli Cohen *out = NULL; 10051a86b377SEli Cohen } 10061a86b377SEli Cohen 10071a86b377SEli Cohen static void free_inout(void *in, void *out) 10081a86b377SEli Cohen { 10091a86b377SEli Cohen kfree(in); 10101a86b377SEli Cohen kfree(out); 10111a86b377SEli Cohen } 10121a86b377SEli Cohen 10131a86b377SEli Cohen /* Two QPs are used by each virtqueue. One is used by the driver and one by 10141a86b377SEli Cohen * firmware. The fw argument indicates whether the subjected QP is the one used 10151a86b377SEli Cohen * by firmware. 10161a86b377SEli Cohen */ 10171a86b377SEli Cohen static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) 10181a86b377SEli Cohen { 10191a86b377SEli Cohen int outlen; 10201a86b377SEli Cohen int inlen; 10211a86b377SEli Cohen void *out; 10221a86b377SEli Cohen void *in; 10231a86b377SEli Cohen int err; 10241a86b377SEli Cohen 10251a86b377SEli Cohen alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); 10261a86b377SEli Cohen if (!in || !out) 10271a86b377SEli Cohen return -ENOMEM; 10281a86b377SEli Cohen 10291a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); 10301a86b377SEli Cohen free_inout(in, out); 10311a86b377SEli Cohen return err; 10321a86b377SEli Cohen } 10331a86b377SEli Cohen 10341a86b377SEli Cohen static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 10351a86b377SEli Cohen { 10361a86b377SEli Cohen int err; 10371a86b377SEli Cohen 10381a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); 10391a86b377SEli Cohen if (err) 10401a86b377SEli Cohen return err; 10411a86b377SEli Cohen 10421a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); 10431a86b377SEli Cohen if (err) 10441a86b377SEli Cohen return err; 10451a86b377SEli Cohen 10461a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); 10471a86b377SEli Cohen if (err) 10481a86b377SEli Cohen return err; 10491a86b377SEli Cohen 10501a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); 10511a86b377SEli Cohen if (err) 10521a86b377SEli Cohen return err; 10531a86b377SEli Cohen 10541a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); 10551a86b377SEli Cohen if (err) 10561a86b377SEli Cohen return err; 10571a86b377SEli Cohen 10581a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); 10591a86b377SEli Cohen if (err) 10601a86b377SEli Cohen return err; 10611a86b377SEli Cohen 10621a86b377SEli Cohen return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); 10631a86b377SEli Cohen } 10641a86b377SEli Cohen 10651a86b377SEli Cohen struct mlx5_virtq_attr { 10661a86b377SEli Cohen u8 state; 10671a86b377SEli Cohen u16 available_index; 1068b35ccebeSEli Cohen u16 used_index; 10691a86b377SEli Cohen }; 10701a86b377SEli Cohen 10711a86b377SEli Cohen static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 10721a86b377SEli Cohen struct mlx5_virtq_attr *attr) 10731a86b377SEli Cohen { 10741a86b377SEli Cohen int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out); 10751a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {}; 10761a86b377SEli Cohen void *out; 10771a86b377SEli Cohen void *obj_context; 10781a86b377SEli Cohen void *cmd_hdr; 10791a86b377SEli Cohen int err; 10801a86b377SEli Cohen 10811a86b377SEli Cohen out = kzalloc(outlen, GFP_KERNEL); 10821a86b377SEli Cohen if (!out) 10831a86b377SEli Cohen return -ENOMEM; 10841a86b377SEli Cohen 10851a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr); 10861a86b377SEli Cohen 10871a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 10881a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 10891a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 10901a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 10911a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen); 10921a86b377SEli Cohen if (err) 10931a86b377SEli Cohen goto err_cmd; 10941a86b377SEli Cohen 10951a86b377SEli Cohen obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context); 10961a86b377SEli Cohen memset(attr, 0, sizeof(*attr)); 10971a86b377SEli Cohen attr->state = MLX5_GET(virtio_net_q_object, obj_context, state); 10981a86b377SEli Cohen attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index); 1099b35ccebeSEli Cohen attr->used_index = MLX5_GET(virtio_net_q_object, obj_context, hw_used_index); 11001a86b377SEli Cohen kfree(out); 11011a86b377SEli Cohen return 0; 11021a86b377SEli Cohen 11031a86b377SEli Cohen err_cmd: 11041a86b377SEli Cohen kfree(out); 11051a86b377SEli Cohen return err; 11061a86b377SEli Cohen } 11071a86b377SEli Cohen 11081a86b377SEli Cohen static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) 11091a86b377SEli Cohen { 11101a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); 11111a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {}; 11121a86b377SEli Cohen void *obj_context; 11131a86b377SEli Cohen void *cmd_hdr; 11141a86b377SEli Cohen void *in; 11151a86b377SEli Cohen int err; 11161a86b377SEli Cohen 11171a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 11181a86b377SEli Cohen if (!in) 11191a86b377SEli Cohen return -ENOMEM; 11201a86b377SEli Cohen 11211a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr); 11221a86b377SEli Cohen 11231a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); 11241a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 11251a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 11261a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 11271a86b377SEli Cohen 11281a86b377SEli Cohen obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context); 11291a86b377SEli Cohen MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select, 11301a86b377SEli Cohen MLX5_VIRTQ_MODIFY_MASK_STATE); 11311a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, state, state); 11321a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 11331a86b377SEli Cohen kfree(in); 11341a86b377SEli Cohen if (!err) 11351a86b377SEli Cohen mvq->fw_state = state; 11361a86b377SEli Cohen 11371a86b377SEli Cohen return err; 11381a86b377SEli Cohen } 11391a86b377SEli Cohen 11401a86b377SEli Cohen static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11411a86b377SEli Cohen { 11421a86b377SEli Cohen u16 idx = mvq->index; 11431a86b377SEli Cohen int err; 11441a86b377SEli Cohen 11451a86b377SEli Cohen if (!mvq->num_ent) 11461a86b377SEli Cohen return 0; 11471a86b377SEli Cohen 114852893733SEli Cohen if (mvq->initialized) 114952893733SEli Cohen return 0; 11501a86b377SEli Cohen 11511a86b377SEli Cohen err = cq_create(ndev, idx, mvq->num_ent); 11521a86b377SEli Cohen if (err) 11531a86b377SEli Cohen return err; 11541a86b377SEli Cohen 11551a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->fwqp); 11561a86b377SEli Cohen if (err) 11571a86b377SEli Cohen goto err_fwqp; 11581a86b377SEli Cohen 11591a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->vqqp); 11601a86b377SEli Cohen if (err) 11611a86b377SEli Cohen goto err_vqqp; 11621a86b377SEli Cohen 11631a86b377SEli Cohen err = connect_qps(ndev, mvq); 11641a86b377SEli Cohen if (err) 11651a86b377SEli Cohen goto err_connect; 11661a86b377SEli Cohen 11671a86b377SEli Cohen err = create_virtqueue(ndev, mvq); 11681a86b377SEli Cohen if (err) 11691a86b377SEli Cohen goto err_connect; 11701a86b377SEli Cohen 11711a86b377SEli Cohen if (mvq->ready) { 11721a86b377SEli Cohen err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 11731a86b377SEli Cohen if (err) { 11741a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", 11751a86b377SEli Cohen idx, err); 11761a86b377SEli Cohen goto err_connect; 11771a86b377SEli Cohen } 11781a86b377SEli Cohen } 11791a86b377SEli Cohen 11801a86b377SEli Cohen mvq->initialized = true; 11811a86b377SEli Cohen return 0; 11821a86b377SEli Cohen 11831a86b377SEli Cohen err_connect: 11841a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 11851a86b377SEli Cohen err_vqqp: 11861a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 11871a86b377SEli Cohen err_fwqp: 11881a86b377SEli Cohen cq_destroy(ndev, idx); 11891a86b377SEli Cohen return err; 11901a86b377SEli Cohen } 11911a86b377SEli Cohen 11921a86b377SEli Cohen static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11931a86b377SEli Cohen { 11941a86b377SEli Cohen struct mlx5_virtq_attr attr; 11951a86b377SEli Cohen 11961a86b377SEli Cohen if (!mvq->initialized) 11971a86b377SEli Cohen return; 11981a86b377SEli Cohen 11991a86b377SEli Cohen if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 12001a86b377SEli Cohen return; 12011a86b377SEli Cohen 12021a86b377SEli Cohen if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) 12031a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); 12043176e974SSi-Wei Liu 12053176e974SSi-Wei Liu if (query_virtqueue(ndev, mvq, &attr)) { 12063176e974SSi-Wei Liu mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); 12073176e974SSi-Wei Liu return; 12083176e974SSi-Wei Liu } 12093176e974SSi-Wei Liu mvq->avail_idx = attr.available_index; 1210bc04d93eSEli Cohen mvq->used_idx = attr.used_index; 12111a86b377SEli Cohen } 12121a86b377SEli Cohen 12131a86b377SEli Cohen static void suspend_vqs(struct mlx5_vdpa_net *ndev) 12141a86b377SEli Cohen { 12151a86b377SEli Cohen int i; 12161a86b377SEli Cohen 12171a86b377SEli Cohen for (i = 0; i < MLX5_MAX_SUPPORTED_VQS; i++) 12181a86b377SEli Cohen suspend_vq(ndev, &ndev->vqs[i]); 12191a86b377SEli Cohen } 12201a86b377SEli Cohen 12211a86b377SEli Cohen static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 12221a86b377SEli Cohen { 12231a86b377SEli Cohen if (!mvq->initialized) 12241a86b377SEli Cohen return; 12251a86b377SEli Cohen 12261a86b377SEli Cohen suspend_vq(ndev, mvq); 12271a86b377SEli Cohen destroy_virtqueue(ndev, mvq); 12281a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 12291a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 12301a86b377SEli Cohen cq_destroy(ndev, mvq->index); 12311a86b377SEli Cohen mvq->initialized = false; 12321a86b377SEli Cohen } 12331a86b377SEli Cohen 12341a86b377SEli Cohen static int create_rqt(struct mlx5_vdpa_net *ndev) 12351a86b377SEli Cohen { 12361a86b377SEli Cohen __be32 *list; 123752893733SEli Cohen int max_rqt; 12381a86b377SEli Cohen void *rqtc; 12391a86b377SEli Cohen int inlen; 12401a86b377SEli Cohen void *in; 12411a86b377SEli Cohen int i, j; 12421a86b377SEli Cohen int err; 12431a86b377SEli Cohen 124452893733SEli Cohen max_rqt = min_t(int, MLX5_MAX_SUPPORTED_VQS / 2, 124552893733SEli Cohen 1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size)); 124652893733SEli Cohen if (max_rqt < 1) 12471a86b377SEli Cohen return -EOPNOTSUPP; 12481a86b377SEli Cohen 124952893733SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + max_rqt * MLX5_ST_SZ_BYTES(rq_num); 12501a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 12511a86b377SEli Cohen if (!in) 12521a86b377SEli Cohen return -ENOMEM; 12531a86b377SEli Cohen 12541a86b377SEli Cohen MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); 12551a86b377SEli Cohen rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 12561a86b377SEli Cohen 12571a86b377SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 125852893733SEli Cohen MLX5_SET(rqtc, rqtc, rqt_max_size, max_rqt); 12591a86b377SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 126052893733SEli Cohen for (i = 0, j = 0; j < max_rqt; j++) { 12611a86b377SEli Cohen if (!ndev->vqs[j].initialized) 12621a86b377SEli Cohen continue; 12631a86b377SEli Cohen 12641a86b377SEli Cohen if (!vq_is_tx(ndev->vqs[j].index)) { 12651a86b377SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j].virtq_id); 12661a86b377SEli Cohen i++; 12671a86b377SEli Cohen } 12681a86b377SEli Cohen } 126952893733SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, i); 12701a86b377SEli Cohen 12711a86b377SEli Cohen err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); 12721a86b377SEli Cohen kfree(in); 12731a86b377SEli Cohen if (err) 12741a86b377SEli Cohen return err; 12751a86b377SEli Cohen 12761a86b377SEli Cohen return 0; 12771a86b377SEli Cohen } 12781a86b377SEli Cohen 127952893733SEli Cohen #define MLX5_MODIFY_RQT_NUM_RQS ((u64)1) 128052893733SEli Cohen 128152893733SEli Cohen static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) 128252893733SEli Cohen { 128352893733SEli Cohen __be32 *list; 128452893733SEli Cohen int max_rqt; 128552893733SEli Cohen void *rqtc; 128652893733SEli Cohen int inlen; 128752893733SEli Cohen void *in; 128852893733SEli Cohen int i, j; 128952893733SEli Cohen int err; 129052893733SEli Cohen 129152893733SEli Cohen max_rqt = min_t(int, ndev->cur_num_vqs / 2, 129252893733SEli Cohen 1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size)); 129352893733SEli Cohen if (max_rqt < 1) 129452893733SEli Cohen return -EOPNOTSUPP; 129552893733SEli Cohen 129652893733SEli Cohen inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + max_rqt * MLX5_ST_SZ_BYTES(rq_num); 129752893733SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 129852893733SEli Cohen if (!in) 129952893733SEli Cohen return -ENOMEM; 130052893733SEli Cohen 130152893733SEli Cohen MLX5_SET(modify_rqt_in, in, uid, ndev->mvdev.res.uid); 130252893733SEli Cohen MLX5_SET64(modify_rqt_in, in, bitmask, MLX5_MODIFY_RQT_NUM_RQS); 130352893733SEli Cohen rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 130452893733SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 130552893733SEli Cohen 130652893733SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 130752893733SEli Cohen for (i = 0, j = 0; j < num; j++) { 130852893733SEli Cohen if (!ndev->vqs[j].initialized) 130952893733SEli Cohen continue; 131052893733SEli Cohen 131152893733SEli Cohen if (!vq_is_tx(ndev->vqs[j].index)) { 131252893733SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j].virtq_id); 131352893733SEli Cohen i++; 131452893733SEli Cohen } 131552893733SEli Cohen } 131652893733SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, i); 131752893733SEli Cohen err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn); 131852893733SEli Cohen kfree(in); 131952893733SEli Cohen if (err) 132052893733SEli Cohen return err; 132152893733SEli Cohen 132252893733SEli Cohen return 0; 132352893733SEli Cohen } 132452893733SEli Cohen 13251a86b377SEli Cohen static void destroy_rqt(struct mlx5_vdpa_net *ndev) 13261a86b377SEli Cohen { 13271a86b377SEli Cohen mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); 13281a86b377SEli Cohen } 13291a86b377SEli Cohen 13301a86b377SEli Cohen static int create_tir(struct mlx5_vdpa_net *ndev) 13311a86b377SEli Cohen { 13321a86b377SEli Cohen #define HASH_IP_L4PORTS \ 13331a86b377SEli Cohen (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \ 13341a86b377SEli Cohen MLX5_HASH_FIELD_SEL_L4_DPORT) 13351a86b377SEli Cohen static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 13361a86b377SEli Cohen 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 13371a86b377SEli Cohen 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 13381a86b377SEli Cohen 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 13391a86b377SEli Cohen 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a }; 13401a86b377SEli Cohen void *rss_key; 13411a86b377SEli Cohen void *outer; 13421a86b377SEli Cohen void *tirc; 13431a86b377SEli Cohen void *in; 13441a86b377SEli Cohen int err; 13451a86b377SEli Cohen 13461a86b377SEli Cohen in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL); 13471a86b377SEli Cohen if (!in) 13481a86b377SEli Cohen return -ENOMEM; 13491a86b377SEli Cohen 13501a86b377SEli Cohen MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); 13511a86b377SEli Cohen tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 13521a86b377SEli Cohen MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 13531a86b377SEli Cohen 13541a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 13551a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ); 13561a86b377SEli Cohen rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 13571a86b377SEli Cohen memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key)); 13581a86b377SEli Cohen 13591a86b377SEli Cohen outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 13601a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); 13611a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); 13621a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS); 13631a86b377SEli Cohen 13641a86b377SEli Cohen MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); 13651a86b377SEli Cohen MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); 13661a86b377SEli Cohen 13671a86b377SEli Cohen err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); 13681a86b377SEli Cohen kfree(in); 13691a86b377SEli Cohen return err; 13701a86b377SEli Cohen } 13711a86b377SEli Cohen 13721a86b377SEli Cohen static void destroy_tir(struct mlx5_vdpa_net *ndev) 13731a86b377SEli Cohen { 13741a86b377SEli Cohen mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); 13751a86b377SEli Cohen } 13761a86b377SEli Cohen 13771a86b377SEli Cohen static int add_fwd_to_tir(struct mlx5_vdpa_net *ndev) 13781a86b377SEli Cohen { 13791a86b377SEli Cohen struct mlx5_flow_destination dest[2] = {}; 13801a86b377SEli Cohen struct mlx5_flow_table_attr ft_attr = {}; 13811a86b377SEli Cohen struct mlx5_flow_act flow_act = {}; 13821a86b377SEli Cohen struct mlx5_flow_namespace *ns; 13831a86b377SEli Cohen int err; 13841a86b377SEli Cohen 13851a86b377SEli Cohen /* for now, one entry, match all, forward to tir */ 13861a86b377SEli Cohen ft_attr.max_fte = 1; 13871a86b377SEli Cohen ft_attr.autogroup.max_num_groups = 1; 13881a86b377SEli Cohen 13891a86b377SEli Cohen ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); 13901a86b377SEli Cohen if (!ns) { 13911a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "get flow namespace\n"); 13921a86b377SEli Cohen return -EOPNOTSUPP; 13931a86b377SEli Cohen } 13941a86b377SEli Cohen 13951a86b377SEli Cohen ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 13961a86b377SEli Cohen if (IS_ERR(ndev->rxft)) 13971a86b377SEli Cohen return PTR_ERR(ndev->rxft); 13981a86b377SEli Cohen 13991a86b377SEli Cohen ndev->rx_counter = mlx5_fc_create(ndev->mvdev.mdev, false); 14001a86b377SEli Cohen if (IS_ERR(ndev->rx_counter)) { 14011a86b377SEli Cohen err = PTR_ERR(ndev->rx_counter); 14021a86b377SEli Cohen goto err_fc; 14031a86b377SEli Cohen } 14041a86b377SEli Cohen 14051a86b377SEli Cohen flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_COUNT; 14061a86b377SEli Cohen dest[0].type = MLX5_FLOW_DESTINATION_TYPE_TIR; 14071a86b377SEli Cohen dest[0].tir_num = ndev->res.tirn; 14081a86b377SEli Cohen dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 14091a86b377SEli Cohen dest[1].counter_id = mlx5_fc_id(ndev->rx_counter); 14101a86b377SEli Cohen ndev->rx_rule = mlx5_add_flow_rules(ndev->rxft, NULL, &flow_act, dest, 2); 14111a86b377SEli Cohen if (IS_ERR(ndev->rx_rule)) { 14121a86b377SEli Cohen err = PTR_ERR(ndev->rx_rule); 14131a86b377SEli Cohen ndev->rx_rule = NULL; 14141a86b377SEli Cohen goto err_rule; 14151a86b377SEli Cohen } 14161a86b377SEli Cohen 14171a86b377SEli Cohen return 0; 14181a86b377SEli Cohen 14191a86b377SEli Cohen err_rule: 14201a86b377SEli Cohen mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter); 14211a86b377SEli Cohen err_fc: 14221a86b377SEli Cohen mlx5_destroy_flow_table(ndev->rxft); 14231a86b377SEli Cohen return err; 14241a86b377SEli Cohen } 14251a86b377SEli Cohen 14261a86b377SEli Cohen static void remove_fwd_to_tir(struct mlx5_vdpa_net *ndev) 14271a86b377SEli Cohen { 14281a86b377SEli Cohen if (!ndev->rx_rule) 14291a86b377SEli Cohen return; 14301a86b377SEli Cohen 14311a86b377SEli Cohen mlx5_del_flow_rules(ndev->rx_rule); 14321a86b377SEli Cohen mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter); 14331a86b377SEli Cohen mlx5_destroy_flow_table(ndev->rxft); 14341a86b377SEli Cohen 14351a86b377SEli Cohen ndev->rx_rule = NULL; 14361a86b377SEli Cohen } 14371a86b377SEli Cohen 14385262912eSEli Cohen static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) 14395262912eSEli Cohen { 14405262912eSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 14415262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 14425262912eSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 14435262912eSEli Cohen struct mlx5_core_dev *pfmdev; 14445262912eSEli Cohen size_t read; 14455262912eSEli Cohen u8 mac[ETH_ALEN]; 14465262912eSEli Cohen 14475262912eSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 14485262912eSEli Cohen switch (cmd) { 14495262912eSEli Cohen case VIRTIO_NET_CTRL_MAC_ADDR_SET: 14505262912eSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)mac, ETH_ALEN); 14515262912eSEli Cohen if (read != ETH_ALEN) 14525262912eSEli Cohen break; 14535262912eSEli Cohen 14545262912eSEli Cohen if (!memcmp(ndev->config.mac, mac, 6)) { 14555262912eSEli Cohen status = VIRTIO_NET_OK; 14565262912eSEli Cohen break; 14575262912eSEli Cohen } 14585262912eSEli Cohen 14595262912eSEli Cohen if (!is_zero_ether_addr(ndev->config.mac)) { 14605262912eSEli Cohen if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 14615262912eSEli Cohen mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n", 14625262912eSEli Cohen ndev->config.mac); 14635262912eSEli Cohen break; 14645262912eSEli Cohen } 14655262912eSEli Cohen } 14665262912eSEli Cohen 14675262912eSEli Cohen if (mlx5_mpfs_add_mac(pfmdev, mac)) { 14685262912eSEli Cohen mlx5_vdpa_warn(mvdev, "failed to insert new MAC %pM into MPFS table\n", 14695262912eSEli Cohen mac); 14705262912eSEli Cohen break; 14715262912eSEli Cohen } 14725262912eSEli Cohen 14735262912eSEli Cohen memcpy(ndev->config.mac, mac, ETH_ALEN); 14745262912eSEli Cohen status = VIRTIO_NET_OK; 14755262912eSEli Cohen break; 14765262912eSEli Cohen 14775262912eSEli Cohen default: 14785262912eSEli Cohen break; 14795262912eSEli Cohen } 14805262912eSEli Cohen 14815262912eSEli Cohen return status; 14825262912eSEli Cohen } 14835262912eSEli Cohen 148452893733SEli Cohen static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps) 148552893733SEli Cohen { 148652893733SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 148752893733SEli Cohen int cur_qps = ndev->cur_num_vqs / 2; 148852893733SEli Cohen int err; 148952893733SEli Cohen int i; 149052893733SEli Cohen 149152893733SEli Cohen if (cur_qps > newqps) { 149252893733SEli Cohen err = modify_rqt(ndev, 2 * newqps); 149352893733SEli Cohen if (err) 149452893733SEli Cohen return err; 149552893733SEli Cohen 149652893733SEli Cohen for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) 149752893733SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 149852893733SEli Cohen 149952893733SEli Cohen ndev->cur_num_vqs = 2 * newqps; 150052893733SEli Cohen } else { 150152893733SEli Cohen ndev->cur_num_vqs = 2 * newqps; 150252893733SEli Cohen for (i = cur_qps * 2; i < 2 * newqps; i++) { 150352893733SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 150452893733SEli Cohen if (err) 150552893733SEli Cohen goto clean_added; 150652893733SEli Cohen } 150752893733SEli Cohen err = modify_rqt(ndev, 2 * newqps); 150852893733SEli Cohen if (err) 150952893733SEli Cohen goto clean_added; 151052893733SEli Cohen } 151152893733SEli Cohen return 0; 151252893733SEli Cohen 151352893733SEli Cohen clean_added: 151452893733SEli Cohen for (--i; i >= cur_qps; --i) 151552893733SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 151652893733SEli Cohen 151752893733SEli Cohen return err; 151852893733SEli Cohen } 151952893733SEli Cohen 152052893733SEli Cohen static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd) 152152893733SEli Cohen { 152252893733SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 152352893733SEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 152452893733SEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 152552893733SEli Cohen struct virtio_net_ctrl_mq mq; 152652893733SEli Cohen size_t read; 152752893733SEli Cohen u16 newqps; 152852893733SEli Cohen 152952893733SEli Cohen switch (cmd) { 153052893733SEli Cohen case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: 153152893733SEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)&mq, sizeof(mq)); 153252893733SEli Cohen if (read != sizeof(mq)) 153352893733SEli Cohen break; 153452893733SEli Cohen 153552893733SEli Cohen newqps = mlx5vdpa16_to_cpu(mvdev, mq.virtqueue_pairs); 153652893733SEli Cohen if (ndev->cur_num_vqs == 2 * newqps) { 153752893733SEli Cohen status = VIRTIO_NET_OK; 153852893733SEli Cohen break; 153952893733SEli Cohen } 154052893733SEli Cohen 154152893733SEli Cohen if (newqps & (newqps - 1)) 154252893733SEli Cohen break; 154352893733SEli Cohen 154452893733SEli Cohen if (!change_num_qps(mvdev, newqps)) 154552893733SEli Cohen status = VIRTIO_NET_OK; 154652893733SEli Cohen 154752893733SEli Cohen break; 154852893733SEli Cohen default: 154952893733SEli Cohen break; 155052893733SEli Cohen } 155152893733SEli Cohen 155252893733SEli Cohen return status; 155352893733SEli Cohen } 155452893733SEli Cohen 15555262912eSEli Cohen static void mlx5_cvq_kick_handler(struct work_struct *work) 15565262912eSEli Cohen { 15575262912eSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 15585262912eSEli Cohen struct virtio_net_ctrl_hdr ctrl; 15595262912eSEli Cohen struct mlx5_ctrl_wq_ent *wqent; 15605262912eSEli Cohen struct mlx5_vdpa_dev *mvdev; 15615262912eSEli Cohen struct mlx5_control_vq *cvq; 15625262912eSEli Cohen struct mlx5_vdpa_net *ndev; 15635262912eSEli Cohen size_t read, write; 15645262912eSEli Cohen int err; 15655262912eSEli Cohen 15665262912eSEli Cohen wqent = container_of(work, struct mlx5_ctrl_wq_ent, work); 15675262912eSEli Cohen mvdev = wqent->mvdev; 15685262912eSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 15695262912eSEli Cohen cvq = &mvdev->cvq; 15705262912eSEli Cohen if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 15715262912eSEli Cohen goto out; 15725262912eSEli Cohen 15735262912eSEli Cohen if (!cvq->ready) 15745262912eSEli Cohen goto out; 15755262912eSEli Cohen 15765262912eSEli Cohen while (true) { 15775262912eSEli Cohen err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head, 15785262912eSEli Cohen GFP_ATOMIC); 15795262912eSEli Cohen if (err <= 0) 15805262912eSEli Cohen break; 15815262912eSEli Cohen 15825262912eSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &ctrl, sizeof(ctrl)); 15835262912eSEli Cohen if (read != sizeof(ctrl)) 15845262912eSEli Cohen break; 15855262912eSEli Cohen 15865262912eSEli Cohen switch (ctrl.class) { 15875262912eSEli Cohen case VIRTIO_NET_CTRL_MAC: 15885262912eSEli Cohen status = handle_ctrl_mac(mvdev, ctrl.cmd); 15895262912eSEli Cohen break; 159052893733SEli Cohen case VIRTIO_NET_CTRL_MQ: 159152893733SEli Cohen status = handle_ctrl_mq(mvdev, ctrl.cmd); 159252893733SEli Cohen break; 15935262912eSEli Cohen 15945262912eSEli Cohen default: 15955262912eSEli Cohen break; 15965262912eSEli Cohen } 15975262912eSEli Cohen 15985262912eSEli Cohen /* Make sure data is written before advancing index */ 15995262912eSEli Cohen smp_wmb(); 16005262912eSEli Cohen 16015262912eSEli Cohen write = vringh_iov_push_iotlb(&cvq->vring, &cvq->wiov, &status, sizeof(status)); 16025262912eSEli Cohen vringh_complete_iotlb(&cvq->vring, cvq->head, write); 16035262912eSEli Cohen vringh_kiov_cleanup(&cvq->riov); 16045262912eSEli Cohen vringh_kiov_cleanup(&cvq->wiov); 16055262912eSEli Cohen 16065262912eSEli Cohen if (vringh_need_notify_iotlb(&cvq->vring)) 16075262912eSEli Cohen vringh_notify(&cvq->vring); 16085262912eSEli Cohen } 16095262912eSEli Cohen out: 16105262912eSEli Cohen kfree(wqent); 16115262912eSEli Cohen } 16125262912eSEli Cohen 16131a86b377SEli Cohen static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) 16141a86b377SEli Cohen { 16151a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 16161a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 16175262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 16185262912eSEli Cohen struct mlx5_ctrl_wq_ent *wqent; 16191a86b377SEli Cohen 1620e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1621e4fc6650SEli Cohen return; 1622e4fc6650SEli Cohen 16235262912eSEli Cohen if (unlikely(is_ctrl_vq_idx(mvdev, idx))) { 16245262912eSEli Cohen if (!mvdev->cvq.ready) 16255262912eSEli Cohen return; 16265262912eSEli Cohen 16275262912eSEli Cohen wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); 16285262912eSEli Cohen if (!wqent) 16295262912eSEli Cohen return; 16305262912eSEli Cohen 16315262912eSEli Cohen wqent->mvdev = mvdev; 16325262912eSEli Cohen INIT_WORK(&wqent->work, mlx5_cvq_kick_handler); 16335262912eSEli Cohen queue_work(mvdev->wq, &wqent->work); 16345262912eSEli Cohen return; 16355262912eSEli Cohen } 16365262912eSEli Cohen 16375262912eSEli Cohen mvq = &ndev->vqs[idx]; 16381a86b377SEli Cohen if (unlikely(!mvq->ready)) 16391a86b377SEli Cohen return; 16401a86b377SEli Cohen 16411a86b377SEli Cohen iowrite16(idx, ndev->mvdev.res.kick_addr); 16421a86b377SEli Cohen } 16431a86b377SEli Cohen 16441a86b377SEli Cohen static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 16451a86b377SEli Cohen u64 driver_area, u64 device_area) 16461a86b377SEli Cohen { 16471a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 16481a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 16495262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 16501a86b377SEli Cohen 1651e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1652e4fc6650SEli Cohen return -EINVAL; 1653e4fc6650SEli Cohen 16545262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 16555262912eSEli Cohen mvdev->cvq.desc_addr = desc_area; 16565262912eSEli Cohen mvdev->cvq.device_addr = device_area; 16575262912eSEli Cohen mvdev->cvq.driver_addr = driver_area; 16585262912eSEli Cohen return 0; 16595262912eSEli Cohen } 16605262912eSEli Cohen 16615262912eSEli Cohen mvq = &ndev->vqs[idx]; 16621a86b377SEli Cohen mvq->desc_addr = desc_area; 16631a86b377SEli Cohen mvq->device_addr = device_area; 16641a86b377SEli Cohen mvq->driver_addr = driver_area; 16651a86b377SEli Cohen return 0; 16661a86b377SEli Cohen } 16671a86b377SEli Cohen 16681a86b377SEli Cohen static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 16691a86b377SEli Cohen { 16701a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 16711a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 16721a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 16731a86b377SEli Cohen 16745262912eSEli Cohen if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 1675e4fc6650SEli Cohen return; 1676e4fc6650SEli Cohen 16771a86b377SEli Cohen mvq = &ndev->vqs[idx]; 16781a86b377SEli Cohen mvq->num_ent = num; 16791a86b377SEli Cohen } 16801a86b377SEli Cohen 16811a86b377SEli Cohen static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 16821a86b377SEli Cohen { 16831a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 16841a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 16851a86b377SEli Cohen 1686db296d25SEli Cohen ndev->event_cbs[idx] = *cb; 16871a86b377SEli Cohen } 16881a86b377SEli Cohen 16895262912eSEli Cohen static void mlx5_cvq_notify(struct vringh *vring) 16905262912eSEli Cohen { 16915262912eSEli Cohen struct mlx5_control_vq *cvq = container_of(vring, struct mlx5_control_vq, vring); 16925262912eSEli Cohen 16935262912eSEli Cohen if (!cvq->event_cb.callback) 16945262912eSEli Cohen return; 16955262912eSEli Cohen 16965262912eSEli Cohen cvq->event_cb.callback(cvq->event_cb.private); 16975262912eSEli Cohen } 16985262912eSEli Cohen 16995262912eSEli Cohen static void set_cvq_ready(struct mlx5_vdpa_dev *mvdev, bool ready) 17005262912eSEli Cohen { 17015262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 17025262912eSEli Cohen 17035262912eSEli Cohen cvq->ready = ready; 17045262912eSEli Cohen if (!ready) 17055262912eSEli Cohen return; 17065262912eSEli Cohen 17075262912eSEli Cohen cvq->vring.notify = mlx5_cvq_notify; 17081a86b377SEli Cohen } 17091a86b377SEli Cohen 17101a86b377SEli Cohen static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 17111a86b377SEli Cohen { 17121a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17131a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17145262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 17151a86b377SEli Cohen 1716759be899SEli Cohen if (!mvdev->actual_features) 1717759be899SEli Cohen return; 1718759be899SEli Cohen 1719e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1720e4fc6650SEli Cohen return; 1721e4fc6650SEli Cohen 17225262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 17235262912eSEli Cohen set_cvq_ready(mvdev, ready); 17245262912eSEli Cohen return; 17255262912eSEli Cohen } 17265262912eSEli Cohen 17275262912eSEli Cohen mvq = &ndev->vqs[idx]; 17281a86b377SEli Cohen if (!ready) 17291a86b377SEli Cohen suspend_vq(ndev, mvq); 17301a86b377SEli Cohen 17311a86b377SEli Cohen mvq->ready = ready; 17321a86b377SEli Cohen } 17331a86b377SEli Cohen 17341a86b377SEli Cohen static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx) 17351a86b377SEli Cohen { 17361a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17371a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17381a86b377SEli Cohen 1739e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1740e4fc6650SEli Cohen return false; 1741e4fc6650SEli Cohen 17425262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) 17435262912eSEli Cohen return mvdev->cvq.ready; 17445262912eSEli Cohen 17455262912eSEli Cohen return ndev->vqs[idx].ready; 17461a86b377SEli Cohen } 17471a86b377SEli Cohen 17481a86b377SEli Cohen static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx, 17491a86b377SEli Cohen const struct vdpa_vq_state *state) 17501a86b377SEli Cohen { 17511a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17521a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17535262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 17541a86b377SEli Cohen 1755e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1756e4fc6650SEli Cohen return -EINVAL; 1757e4fc6650SEli Cohen 17585262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 17595262912eSEli Cohen mvdev->cvq.vring.last_avail_idx = state->split.avail_index; 17605262912eSEli Cohen return 0; 17615262912eSEli Cohen } 17625262912eSEli Cohen 17635262912eSEli Cohen mvq = &ndev->vqs[idx]; 17641a86b377SEli Cohen if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) { 17651a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "can't modify available index\n"); 17661a86b377SEli Cohen return -EINVAL; 17671a86b377SEli Cohen } 17681a86b377SEli Cohen 1769530a5678SJason Wang mvq->used_idx = state->split.avail_index; 1770530a5678SJason Wang mvq->avail_idx = state->split.avail_index; 17711a86b377SEli Cohen return 0; 17721a86b377SEli Cohen } 17731a86b377SEli Cohen 17741a86b377SEli Cohen static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 17751a86b377SEli Cohen { 17761a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17771a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17785262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 17791a86b377SEli Cohen struct mlx5_virtq_attr attr; 17801a86b377SEli Cohen int err; 17811a86b377SEli Cohen 1782e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1783e4fc6650SEli Cohen return -EINVAL; 1784e4fc6650SEli Cohen 17855262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 17865262912eSEli Cohen state->split.avail_index = mvdev->cvq.vring.last_avail_idx; 17875262912eSEli Cohen return 0; 17885262912eSEli Cohen } 17895262912eSEli Cohen 17905262912eSEli Cohen mvq = &ndev->vqs[idx]; 17913176e974SSi-Wei Liu /* If the virtq object was destroyed, use the value saved at 17923176e974SSi-Wei Liu * the last minute of suspend_vq. This caters for userspace 17933176e974SSi-Wei Liu * that cares about emulating the index after vq is stopped. 17943176e974SSi-Wei Liu */ 17953176e974SSi-Wei Liu if (!mvq->initialized) { 1796bc04d93eSEli Cohen /* Firmware returns a wrong value for the available index. 1797bc04d93eSEli Cohen * Since both values should be identical, we take the value of 1798bc04d93eSEli Cohen * used_idx which is reported correctly. 1799bc04d93eSEli Cohen */ 1800530a5678SJason Wang state->split.avail_index = mvq->used_idx; 18013176e974SSi-Wei Liu return 0; 18023176e974SSi-Wei Liu } 18031a86b377SEli Cohen 18041a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 18051a86b377SEli Cohen if (err) { 18061a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n"); 18071a86b377SEli Cohen return err; 18081a86b377SEli Cohen } 1809530a5678SJason Wang state->split.avail_index = attr.used_index; 18101a86b377SEli Cohen return 0; 18111a86b377SEli Cohen } 18121a86b377SEli Cohen 18131a86b377SEli Cohen static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) 18141a86b377SEli Cohen { 18151a86b377SEli Cohen return PAGE_SIZE; 18161a86b377SEli Cohen } 18171a86b377SEli Cohen 18181a86b377SEli Cohen enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9, 18191a86b377SEli Cohen MLX5_VIRTIO_NET_F_CSUM = 1 << 10, 18201a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11, 18211a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12, 18221a86b377SEli Cohen }; 18231a86b377SEli Cohen 18241a86b377SEli Cohen static u64 mlx_to_vritio_features(u16 dev_features) 18251a86b377SEli Cohen { 18261a86b377SEli Cohen u64 result = 0; 18271a86b377SEli Cohen 18281a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM) 1829cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM); 18301a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_CSUM) 1831cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_CSUM); 18321a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6) 1833cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6); 18341a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4) 1835cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4); 18361a86b377SEli Cohen 18371a86b377SEli Cohen return result; 18381a86b377SEli Cohen } 18391a86b377SEli Cohen 18401a86b377SEli Cohen static u64 mlx5_vdpa_get_features(struct vdpa_device *vdev) 18411a86b377SEli Cohen { 18421a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 18431a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 18441a86b377SEli Cohen u16 dev_features; 18451a86b377SEli Cohen 18461a86b377SEli Cohen dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, device_features_bits_mask); 18475262912eSEli Cohen ndev->mvdev.mlx_features |= mlx_to_vritio_features(dev_features); 18481a86b377SEli Cohen if (MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, virtio_version_1_0)) 1849cbb52359SNathan Chancellor ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_F_VERSION_1); 1850cbb52359SNathan Chancellor ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM); 18515262912eSEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VQ); 18525262912eSEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR); 185352893733SEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MQ); 18545262912eSEli Cohen 18551a86b377SEli Cohen print_features(mvdev, ndev->mvdev.mlx_features, false); 18561a86b377SEli Cohen return ndev->mvdev.mlx_features; 18571a86b377SEli Cohen } 18581a86b377SEli Cohen 18591a86b377SEli Cohen static int verify_min_features(struct mlx5_vdpa_dev *mvdev, u64 features) 18601a86b377SEli Cohen { 1861cbb52359SNathan Chancellor if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) 18621a86b377SEli Cohen return -EOPNOTSUPP; 18631a86b377SEli Cohen 18641a86b377SEli Cohen return 0; 18651a86b377SEli Cohen } 18661a86b377SEli Cohen 1867ae0428deSEli Cohen static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev) 18681a86b377SEli Cohen { 1869ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 18705262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 18711a86b377SEli Cohen int err; 18721a86b377SEli Cohen int i; 18731a86b377SEli Cohen 1874ae0428deSEli Cohen for (i = 0; i < 2 * mlx5_vdpa_max_qps(mvdev->max_vqs); i++) { 18751a86b377SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 18761a86b377SEli Cohen if (err) 18771a86b377SEli Cohen goto err_vq; 18781a86b377SEli Cohen } 18791a86b377SEli Cohen 18805262912eSEli Cohen if (mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) { 18815262912eSEli Cohen err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features, 18825262912eSEli Cohen MLX5_CVQ_MAX_ENT, false, 18835262912eSEli Cohen (struct vring_desc *)(uintptr_t)cvq->desc_addr, 18845262912eSEli Cohen (struct vring_avail *)(uintptr_t)cvq->driver_addr, 18855262912eSEli Cohen (struct vring_used *)(uintptr_t)cvq->device_addr); 18865262912eSEli Cohen if (err) 18875262912eSEli Cohen goto err_vq; 18885262912eSEli Cohen } 18895262912eSEli Cohen 18901a86b377SEli Cohen return 0; 18911a86b377SEli Cohen 18921a86b377SEli Cohen err_vq: 18931a86b377SEli Cohen for (--i; i >= 0; i--) 18941a86b377SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 18951a86b377SEli Cohen 18961a86b377SEli Cohen return err; 18971a86b377SEli Cohen } 18981a86b377SEli Cohen 18991a86b377SEli Cohen static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) 19001a86b377SEli Cohen { 19011a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 19021a86b377SEli Cohen int i; 19031a86b377SEli Cohen 19041a86b377SEli Cohen for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { 19051a86b377SEli Cohen mvq = &ndev->vqs[i]; 19061a86b377SEli Cohen if (!mvq->initialized) 19071a86b377SEli Cohen continue; 19081a86b377SEli Cohen 19091a86b377SEli Cohen teardown_vq(ndev, mvq); 19101a86b377SEli Cohen } 19111a86b377SEli Cohen } 19121a86b377SEli Cohen 1913e4fc6650SEli Cohen static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) 19148a7c3213SMichael S. Tsirkin { 1915e4fc6650SEli Cohen if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_CTRL_VQ)) { 1916e4fc6650SEli Cohen if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) { 1917e4fc6650SEli Cohen /* MQ supported. CVQ index is right above the last data virtqueue's */ 1918e4fc6650SEli Cohen mvdev->max_idx = mvdev->max_vqs; 1919e4fc6650SEli Cohen } else { 1920e4fc6650SEli Cohen /* Only CVQ supportted. data virtqueues occupy indices 0 and 1. 1921e4fc6650SEli Cohen * CVQ gets index 2 1922e4fc6650SEli Cohen */ 1923e4fc6650SEli Cohen mvdev->max_idx = 2; 19248a7c3213SMichael S. Tsirkin } 1925e4fc6650SEli Cohen } else { 1926e4fc6650SEli Cohen /* Two data virtqueues only: one for rx and one for tx */ 1927e4fc6650SEli Cohen mvdev->max_idx = 1; 1928e4fc6650SEli Cohen } 192936bdcf31SEli Cohen } 193036bdcf31SEli Cohen 19311a86b377SEli Cohen static int mlx5_vdpa_set_features(struct vdpa_device *vdev, u64 features) 19321a86b377SEli Cohen { 19331a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19341a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19351a86b377SEli Cohen int err; 19361a86b377SEli Cohen 19371a86b377SEli Cohen print_features(mvdev, features, true); 19381a86b377SEli Cohen 19391a86b377SEli Cohen err = verify_min_features(mvdev, features); 19401a86b377SEli Cohen if (err) 19411a86b377SEli Cohen return err; 19421a86b377SEli Cohen 19431a86b377SEli Cohen ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; 1944e4fc6650SEli Cohen update_cvq_info(mvdev); 19451a86b377SEli Cohen return err; 19461a86b377SEli Cohen } 19471a86b377SEli Cohen 19481a86b377SEli Cohen static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 19491a86b377SEli Cohen { 19501a86b377SEli Cohen /* not implemented */ 19511a86b377SEli Cohen mlx5_vdpa_warn(to_mvdev(vdev), "set config callback not supported\n"); 19521a86b377SEli Cohen } 19531a86b377SEli Cohen 19541a86b377SEli Cohen #define MLX5_VDPA_MAX_VQ_ENTRIES 256 19551a86b377SEli Cohen static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev) 19561a86b377SEli Cohen { 19571a86b377SEli Cohen return MLX5_VDPA_MAX_VQ_ENTRIES; 19581a86b377SEli Cohen } 19591a86b377SEli Cohen 19601a86b377SEli Cohen static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev) 19611a86b377SEli Cohen { 19621a86b377SEli Cohen return VIRTIO_ID_NET; 19631a86b377SEli Cohen } 19641a86b377SEli Cohen 19651a86b377SEli Cohen static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev) 19661a86b377SEli Cohen { 19671a86b377SEli Cohen return PCI_VENDOR_ID_MELLANOX; 19681a86b377SEli Cohen } 19691a86b377SEli Cohen 19701a86b377SEli Cohen static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev) 19711a86b377SEli Cohen { 19721a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19731a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19741a86b377SEli Cohen 19751a86b377SEli Cohen print_status(mvdev, ndev->mvdev.status, false); 19761a86b377SEli Cohen return ndev->mvdev.status; 19771a86b377SEli Cohen } 19781a86b377SEli Cohen 19791a86b377SEli Cohen static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 19801a86b377SEli Cohen { 19811a86b377SEli Cohen struct mlx5_vq_restore_info *ri = &mvq->ri; 198252893733SEli Cohen struct mlx5_virtq_attr attr = {}; 19831a86b377SEli Cohen int err; 19841a86b377SEli Cohen 198552893733SEli Cohen if (mvq->initialized) { 19861a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 19871a86b377SEli Cohen if (err) 19881a86b377SEli Cohen return err; 198952893733SEli Cohen } 19901a86b377SEli Cohen 19911a86b377SEli Cohen ri->avail_index = attr.available_index; 1992b35ccebeSEli Cohen ri->used_index = attr.used_index; 19931a86b377SEli Cohen ri->ready = mvq->ready; 19941a86b377SEli Cohen ri->num_ent = mvq->num_ent; 19951a86b377SEli Cohen ri->desc_addr = mvq->desc_addr; 19961a86b377SEli Cohen ri->device_addr = mvq->device_addr; 19971a86b377SEli Cohen ri->driver_addr = mvq->driver_addr; 19981a86b377SEli Cohen ri->restore = true; 19991a86b377SEli Cohen return 0; 20001a86b377SEli Cohen } 20011a86b377SEli Cohen 20021a86b377SEli Cohen static int save_channels_info(struct mlx5_vdpa_net *ndev) 20031a86b377SEli Cohen { 20041a86b377SEli Cohen int i; 20051a86b377SEli Cohen 20061a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 20071a86b377SEli Cohen memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); 20081a86b377SEli Cohen save_channel_info(ndev, &ndev->vqs[i]); 20091a86b377SEli Cohen } 20101a86b377SEli Cohen return 0; 20111a86b377SEli Cohen } 20121a86b377SEli Cohen 20131a86b377SEli Cohen static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) 20141a86b377SEli Cohen { 20151a86b377SEli Cohen int i; 20161a86b377SEli Cohen 20171a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 20181a86b377SEli Cohen memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 20191a86b377SEli Cohen } 20201a86b377SEli Cohen 20211a86b377SEli Cohen static void restore_channels_info(struct mlx5_vdpa_net *ndev) 20221a86b377SEli Cohen { 20231a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 20241a86b377SEli Cohen struct mlx5_vq_restore_info *ri; 20251a86b377SEli Cohen int i; 20261a86b377SEli Cohen 20271a86b377SEli Cohen mlx5_clear_vqs(ndev); 20281a86b377SEli Cohen init_mvqs(ndev); 20291a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 20301a86b377SEli Cohen mvq = &ndev->vqs[i]; 20311a86b377SEli Cohen ri = &mvq->ri; 20321a86b377SEli Cohen if (!ri->restore) 20331a86b377SEli Cohen continue; 20341a86b377SEli Cohen 20351a86b377SEli Cohen mvq->avail_idx = ri->avail_index; 2036b35ccebeSEli Cohen mvq->used_idx = ri->used_index; 20371a86b377SEli Cohen mvq->ready = ri->ready; 20381a86b377SEli Cohen mvq->num_ent = ri->num_ent; 20391a86b377SEli Cohen mvq->desc_addr = ri->desc_addr; 20401a86b377SEli Cohen mvq->device_addr = ri->device_addr; 20411a86b377SEli Cohen mvq->driver_addr = ri->driver_addr; 20421a86b377SEli Cohen } 20431a86b377SEli Cohen } 20441a86b377SEli Cohen 2045ae0428deSEli Cohen static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) 20461a86b377SEli Cohen { 2047ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20481a86b377SEli Cohen int err; 20491a86b377SEli Cohen 20501a86b377SEli Cohen suspend_vqs(ndev); 20511a86b377SEli Cohen err = save_channels_info(ndev); 20521a86b377SEli Cohen if (err) 20531a86b377SEli Cohen goto err_mr; 20541a86b377SEli Cohen 20551a86b377SEli Cohen teardown_driver(ndev); 2056ae0428deSEli Cohen mlx5_vdpa_destroy_mr(mvdev); 2057ae0428deSEli Cohen err = mlx5_vdpa_create_mr(mvdev, iotlb); 20581a86b377SEli Cohen if (err) 20591a86b377SEli Cohen goto err_mr; 20601a86b377SEli Cohen 2061ae0428deSEli Cohen if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 20621897f0b6SEli Cohen return 0; 20631897f0b6SEli Cohen 20641a86b377SEli Cohen restore_channels_info(ndev); 2065ae0428deSEli Cohen err = setup_driver(mvdev); 20661a86b377SEli Cohen if (err) 20671a86b377SEli Cohen goto err_setup; 20681a86b377SEli Cohen 20691a86b377SEli Cohen return 0; 20701a86b377SEli Cohen 20711a86b377SEli Cohen err_setup: 2072ae0428deSEli Cohen mlx5_vdpa_destroy_mr(mvdev); 20731a86b377SEli Cohen err_mr: 20741a86b377SEli Cohen return err; 20751a86b377SEli Cohen } 20761a86b377SEli Cohen 2077ae0428deSEli Cohen static int setup_driver(struct mlx5_vdpa_dev *mvdev) 20781a86b377SEli Cohen { 2079ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20801a86b377SEli Cohen int err; 20811a86b377SEli Cohen 20821a86b377SEli Cohen mutex_lock(&ndev->reslock); 20831a86b377SEli Cohen if (ndev->setup) { 2084ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n"); 20851a86b377SEli Cohen err = 0; 20861a86b377SEli Cohen goto out; 20871a86b377SEli Cohen } 2088ae0428deSEli Cohen err = setup_virtqueues(mvdev); 20891a86b377SEli Cohen if (err) { 2090ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "setup_virtqueues\n"); 20911a86b377SEli Cohen goto out; 20921a86b377SEli Cohen } 20931a86b377SEli Cohen 20941a86b377SEli Cohen err = create_rqt(ndev); 20951a86b377SEli Cohen if (err) { 2096ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "create_rqt\n"); 20971a86b377SEli Cohen goto err_rqt; 20981a86b377SEli Cohen } 20991a86b377SEli Cohen 21001a86b377SEli Cohen err = create_tir(ndev); 21011a86b377SEli Cohen if (err) { 2102ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "create_tir\n"); 21031a86b377SEli Cohen goto err_tir; 21041a86b377SEli Cohen } 21051a86b377SEli Cohen 21061a86b377SEli Cohen err = add_fwd_to_tir(ndev); 21071a86b377SEli Cohen if (err) { 2108ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "add_fwd_to_tir\n"); 21091a86b377SEli Cohen goto err_fwd; 21101a86b377SEli Cohen } 21111a86b377SEli Cohen ndev->setup = true; 21121a86b377SEli Cohen mutex_unlock(&ndev->reslock); 21131a86b377SEli Cohen 21141a86b377SEli Cohen return 0; 21151a86b377SEli Cohen 21161a86b377SEli Cohen err_fwd: 21171a86b377SEli Cohen destroy_tir(ndev); 21181a86b377SEli Cohen err_tir: 21191a86b377SEli Cohen destroy_rqt(ndev); 21201a86b377SEli Cohen err_rqt: 21211a86b377SEli Cohen teardown_virtqueues(ndev); 21221a86b377SEli Cohen out: 21231a86b377SEli Cohen mutex_unlock(&ndev->reslock); 21241a86b377SEli Cohen return err; 21251a86b377SEli Cohen } 21261a86b377SEli Cohen 21271a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev) 21281a86b377SEli Cohen { 21291a86b377SEli Cohen mutex_lock(&ndev->reslock); 21301a86b377SEli Cohen if (!ndev->setup) 21311a86b377SEli Cohen goto out; 21321a86b377SEli Cohen 21331a86b377SEli Cohen remove_fwd_to_tir(ndev); 21341a86b377SEli Cohen destroy_tir(ndev); 21351a86b377SEli Cohen destroy_rqt(ndev); 21361a86b377SEli Cohen teardown_virtqueues(ndev); 21371a86b377SEli Cohen ndev->setup = false; 21381a86b377SEli Cohen out: 21391a86b377SEli Cohen mutex_unlock(&ndev->reslock); 21401a86b377SEli Cohen } 21411a86b377SEli Cohen 2142e3aadf2eSEli Cohen static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) 2143e3aadf2eSEli Cohen { 2144e3aadf2eSEli Cohen int i; 2145e3aadf2eSEli Cohen 2146e3aadf2eSEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 2147e3aadf2eSEli Cohen ndev->vqs[i].ready = false; 2148ef12e4bfSEli Cohen 2149ef12e4bfSEli Cohen ndev->mvdev.cvq.ready = false; 2150e3aadf2eSEli Cohen } 2151e3aadf2eSEli Cohen 21521a86b377SEli Cohen static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) 21531a86b377SEli Cohen { 21541a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 21551a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 21561a86b377SEli Cohen int err; 21571a86b377SEli Cohen 21581a86b377SEli Cohen print_status(mvdev, status, true); 21591a86b377SEli Cohen 21601a86b377SEli Cohen if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { 21611a86b377SEli Cohen if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 2162ae0428deSEli Cohen err = setup_driver(mvdev); 21631a86b377SEli Cohen if (err) { 21641a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); 21651a86b377SEli Cohen goto err_setup; 21661a86b377SEli Cohen } 21671a86b377SEli Cohen } else { 21681a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); 21691a86b377SEli Cohen return; 21701a86b377SEli Cohen } 21711a86b377SEli Cohen } 21721a86b377SEli Cohen 21731a86b377SEli Cohen ndev->mvdev.status = status; 21741a86b377SEli Cohen return; 21751a86b377SEli Cohen 21761a86b377SEli Cohen err_setup: 21771a86b377SEli Cohen mlx5_vdpa_destroy_mr(&ndev->mvdev); 21781a86b377SEli Cohen ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; 21791a86b377SEli Cohen } 21801a86b377SEli Cohen 21810686082dSXie Yongji static int mlx5_vdpa_reset(struct vdpa_device *vdev) 21820686082dSXie Yongji { 21830686082dSXie Yongji struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 21840686082dSXie Yongji struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 21850686082dSXie Yongji 21860686082dSXie Yongji print_status(mvdev, 0, true); 21870686082dSXie Yongji mlx5_vdpa_info(mvdev, "performing device reset\n"); 21880686082dSXie Yongji teardown_driver(ndev); 21890686082dSXie Yongji clear_vqs_ready(ndev); 21900686082dSXie Yongji mlx5_vdpa_destroy_mr(&ndev->mvdev); 21910686082dSXie Yongji ndev->mvdev.status = 0; 21920686082dSXie Yongji ndev->mvdev.mlx_features = 0; 21930686082dSXie Yongji memset(ndev->event_cbs, 0, sizeof(ndev->event_cbs)); 21940686082dSXie Yongji ndev->mvdev.actual_features = 0; 21950686082dSXie Yongji ++mvdev->generation; 21960686082dSXie Yongji if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 21970686082dSXie Yongji if (mlx5_vdpa_create_mr(mvdev, NULL)) 21980686082dSXie Yongji mlx5_vdpa_warn(mvdev, "create MR failed\n"); 21990686082dSXie Yongji } 22000686082dSXie Yongji 22010686082dSXie Yongji return 0; 22020686082dSXie Yongji } 22030686082dSXie Yongji 2204442706f9SStefano Garzarella static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev) 2205442706f9SStefano Garzarella { 2206442706f9SStefano Garzarella return sizeof(struct virtio_net_config); 2207442706f9SStefano Garzarella } 2208442706f9SStefano Garzarella 22091a86b377SEli Cohen static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, 22101a86b377SEli Cohen unsigned int len) 22111a86b377SEli Cohen { 22121a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22131a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 22141a86b377SEli Cohen 2215dcfde163SStefano Garzarella if (offset + len <= sizeof(struct virtio_net_config)) 22162874211fSDan Carpenter memcpy(buf, (u8 *)&ndev->config + offset, len); 22171a86b377SEli Cohen } 22181a86b377SEli Cohen 22191a86b377SEli Cohen static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf, 22201a86b377SEli Cohen unsigned int len) 22211a86b377SEli Cohen { 22221a86b377SEli Cohen /* not supported */ 22231a86b377SEli Cohen } 22241a86b377SEli Cohen 22251a86b377SEli Cohen static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) 22261a86b377SEli Cohen { 22271a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22281a86b377SEli Cohen 22291a86b377SEli Cohen return mvdev->generation; 22301a86b377SEli Cohen } 22311a86b377SEli Cohen 22321a86b377SEli Cohen static int mlx5_vdpa_set_map(struct vdpa_device *vdev, struct vhost_iotlb *iotlb) 22331a86b377SEli Cohen { 22341a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22351a86b377SEli Cohen bool change_map; 22361a86b377SEli Cohen int err; 22371a86b377SEli Cohen 22381a86b377SEli Cohen err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map); 22391a86b377SEli Cohen if (err) { 22401a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); 22411a86b377SEli Cohen return err; 22421a86b377SEli Cohen } 22431a86b377SEli Cohen 22441a86b377SEli Cohen if (change_map) 2245ae0428deSEli Cohen return mlx5_vdpa_change_map(mvdev, iotlb); 22461a86b377SEli Cohen 22471a86b377SEli Cohen return 0; 22481a86b377SEli Cohen } 22491a86b377SEli Cohen 22501a86b377SEli Cohen static void mlx5_vdpa_free(struct vdpa_device *vdev) 22511a86b377SEli Cohen { 22521a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22537c9f131fSEli Cohen struct mlx5_core_dev *pfmdev; 22541a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 22551a86b377SEli Cohen 22561a86b377SEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 22571a86b377SEli Cohen 22581a86b377SEli Cohen free_resources(ndev); 22596f5312f8SEli Cohen mlx5_vdpa_destroy_mr(mvdev); 22607c9f131fSEli Cohen if (!is_zero_ether_addr(ndev->config.mac)) { 22617c9f131fSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 22627c9f131fSEli Cohen mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); 22637c9f131fSEli Cohen } 22641a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 22651a86b377SEli Cohen mutex_destroy(&ndev->reslock); 22661a86b377SEli Cohen } 22671a86b377SEli Cohen 22681a86b377SEli Cohen static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx) 22691a86b377SEli Cohen { 2270b57c46cbSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22711a86b377SEli Cohen struct vdpa_notification_area ret = {}; 2272b57c46cbSEli Cohen struct mlx5_vdpa_net *ndev; 2273b57c46cbSEli Cohen phys_addr_t addr; 22741a86b377SEli Cohen 22755262912eSEli Cohen if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2276e4fc6650SEli Cohen return ret; 2277e4fc6650SEli Cohen 2278b57c46cbSEli Cohen /* If SF BAR size is smaller than PAGE_SIZE, do not use direct 2279b57c46cbSEli Cohen * notification to avoid the risk of mapping pages that contain BAR of more 2280b57c46cbSEli Cohen * than one SF 2281b57c46cbSEli Cohen */ 2282b57c46cbSEli Cohen if (MLX5_CAP_GEN(mvdev->mdev, log_min_sf_size) + 12 < PAGE_SHIFT) 2283b57c46cbSEli Cohen return ret; 2284b57c46cbSEli Cohen 2285b57c46cbSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 2286b57c46cbSEli Cohen addr = (phys_addr_t)ndev->mvdev.res.phys_kick_addr; 2287b57c46cbSEli Cohen ret.addr = addr; 2288b57c46cbSEli Cohen ret.size = PAGE_SIZE; 22891a86b377SEli Cohen return ret; 22901a86b377SEli Cohen } 22911a86b377SEli Cohen 22921a86b377SEli Cohen static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) 22931a86b377SEli Cohen { 22941a86b377SEli Cohen return -EOPNOTSUPP; 22951a86b377SEli Cohen } 22961a86b377SEli Cohen 22971a86b377SEli Cohen static const struct vdpa_config_ops mlx5_vdpa_ops = { 22981a86b377SEli Cohen .set_vq_address = mlx5_vdpa_set_vq_address, 22991a86b377SEli Cohen .set_vq_num = mlx5_vdpa_set_vq_num, 23001a86b377SEli Cohen .kick_vq = mlx5_vdpa_kick_vq, 23011a86b377SEli Cohen .set_vq_cb = mlx5_vdpa_set_vq_cb, 23021a86b377SEli Cohen .set_vq_ready = mlx5_vdpa_set_vq_ready, 23031a86b377SEli Cohen .get_vq_ready = mlx5_vdpa_get_vq_ready, 23041a86b377SEli Cohen .set_vq_state = mlx5_vdpa_set_vq_state, 23051a86b377SEli Cohen .get_vq_state = mlx5_vdpa_get_vq_state, 23061a86b377SEli Cohen .get_vq_notification = mlx5_get_vq_notification, 23071a86b377SEli Cohen .get_vq_irq = mlx5_get_vq_irq, 23081a86b377SEli Cohen .get_vq_align = mlx5_vdpa_get_vq_align, 23091a86b377SEli Cohen .get_features = mlx5_vdpa_get_features, 23101a86b377SEli Cohen .set_features = mlx5_vdpa_set_features, 23111a86b377SEli Cohen .set_config_cb = mlx5_vdpa_set_config_cb, 23121a86b377SEli Cohen .get_vq_num_max = mlx5_vdpa_get_vq_num_max, 23131a86b377SEli Cohen .get_device_id = mlx5_vdpa_get_device_id, 23141a86b377SEli Cohen .get_vendor_id = mlx5_vdpa_get_vendor_id, 23151a86b377SEli Cohen .get_status = mlx5_vdpa_get_status, 23161a86b377SEli Cohen .set_status = mlx5_vdpa_set_status, 23170686082dSXie Yongji .reset = mlx5_vdpa_reset, 2318442706f9SStefano Garzarella .get_config_size = mlx5_vdpa_get_config_size, 23191a86b377SEli Cohen .get_config = mlx5_vdpa_get_config, 23201a86b377SEli Cohen .set_config = mlx5_vdpa_set_config, 23211a86b377SEli Cohen .get_generation = mlx5_vdpa_get_generation, 23221a86b377SEli Cohen .set_map = mlx5_vdpa_set_map, 23231a86b377SEli Cohen .free = mlx5_vdpa_free, 23241a86b377SEli Cohen }; 23251a86b377SEli Cohen 2326d084d996SSi-Wei Liu static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) 2327d084d996SSi-Wei Liu { 2328d084d996SSi-Wei Liu u16 hw_mtu; 2329d084d996SSi-Wei Liu int err; 2330d084d996SSi-Wei Liu 2331d084d996SSi-Wei Liu err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2332d084d996SSi-Wei Liu if (err) 2333d084d996SSi-Wei Liu return err; 2334d084d996SSi-Wei Liu 2335d084d996SSi-Wei Liu *mtu = hw_mtu - MLX5V_ETH_HARD_MTU; 2336d084d996SSi-Wei Liu return 0; 2337d084d996SSi-Wei Liu } 2338d084d996SSi-Wei Liu 23391a86b377SEli Cohen static int alloc_resources(struct mlx5_vdpa_net *ndev) 23401a86b377SEli Cohen { 23411a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 23421a86b377SEli Cohen int err; 23431a86b377SEli Cohen 23441a86b377SEli Cohen if (res->valid) { 23451a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); 23461a86b377SEli Cohen return -EEXIST; 23471a86b377SEli Cohen } 23481a86b377SEli Cohen 23491a86b377SEli Cohen err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); 23501a86b377SEli Cohen if (err) 23511a86b377SEli Cohen return err; 23521a86b377SEli Cohen 23531a86b377SEli Cohen err = create_tis(ndev); 23541a86b377SEli Cohen if (err) 23551a86b377SEli Cohen goto err_tis; 23561a86b377SEli Cohen 23571a86b377SEli Cohen res->valid = true; 23581a86b377SEli Cohen 23591a86b377SEli Cohen return 0; 23601a86b377SEli Cohen 23611a86b377SEli Cohen err_tis: 23621a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 23631a86b377SEli Cohen return err; 23641a86b377SEli Cohen } 23651a86b377SEli Cohen 23661a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev) 23671a86b377SEli Cohen { 23681a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 23691a86b377SEli Cohen 23701a86b377SEli Cohen if (!res->valid) 23711a86b377SEli Cohen return; 23721a86b377SEli Cohen 23731a86b377SEli Cohen destroy_tis(ndev); 23741a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 23751a86b377SEli Cohen res->valid = false; 23761a86b377SEli Cohen } 23771a86b377SEli Cohen 23781a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev) 23791a86b377SEli Cohen { 23801a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 23811a86b377SEli Cohen int i; 23821a86b377SEli Cohen 23831a86b377SEli Cohen for (i = 0; i < 2 * mlx5_vdpa_max_qps(ndev->mvdev.max_vqs); ++i) { 23841a86b377SEli Cohen mvq = &ndev->vqs[i]; 23851a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 23861a86b377SEli Cohen mvq->index = i; 23871a86b377SEli Cohen mvq->ndev = ndev; 23881a86b377SEli Cohen mvq->fwqp.fw = true; 23891a86b377SEli Cohen } 23901a86b377SEli Cohen for (; i < ndev->mvdev.max_vqs; i++) { 23911a86b377SEli Cohen mvq = &ndev->vqs[i]; 23921a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 23931a86b377SEli Cohen mvq->index = i; 23941a86b377SEli Cohen mvq->ndev = ndev; 23951a86b377SEli Cohen } 23961a86b377SEli Cohen } 23971a86b377SEli Cohen 239858926c8aSEli Cohen struct mlx5_vdpa_mgmtdev { 239958926c8aSEli Cohen struct vdpa_mgmt_dev mgtdev; 240058926c8aSEli Cohen struct mlx5_adev *madev; 240158926c8aSEli Cohen struct mlx5_vdpa_net *ndev; 240258926c8aSEli Cohen }; 240358926c8aSEli Cohen 240458926c8aSEli Cohen static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name) 24051a86b377SEli Cohen { 240658926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 24071a86b377SEli Cohen struct virtio_net_config *config; 24087c9f131fSEli Cohen struct mlx5_core_dev *pfmdev; 24091a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev; 24101a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 241158926c8aSEli Cohen struct mlx5_core_dev *mdev; 24121a86b377SEli Cohen u32 max_vqs; 2413*246fd1caSEli Cohen u16 mtu; 24141a86b377SEli Cohen int err; 24151a86b377SEli Cohen 241658926c8aSEli Cohen if (mgtdev->ndev) 241758926c8aSEli Cohen return -ENOSPC; 241858926c8aSEli Cohen 241958926c8aSEli Cohen mdev = mgtdev->madev->mdev; 2420879753c8SEli Cohen if (!(MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_queue_type) & 2421879753c8SEli Cohen MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)) { 2422879753c8SEli Cohen dev_warn(mdev->device, "missing support for split virtqueues\n"); 2423879753c8SEli Cohen return -EOPNOTSUPP; 2424879753c8SEli Cohen } 2425879753c8SEli Cohen 24261a86b377SEli Cohen /* we save one virtqueue for control virtqueue should we require it */ 24271a86b377SEli Cohen max_vqs = MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues); 24281a86b377SEli Cohen max_vqs = min_t(u32, max_vqs, MLX5_MAX_SUPPORTED_VQS); 24291a86b377SEli Cohen 24301a86b377SEli Cohen ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, 2431d8945ec4SXie Yongji name, false); 24321a86b377SEli Cohen if (IS_ERR(ndev)) 243374c9729dSLeon Romanovsky return PTR_ERR(ndev); 24341a86b377SEli Cohen 24351a86b377SEli Cohen ndev->mvdev.max_vqs = max_vqs; 24361a86b377SEli Cohen mvdev = &ndev->mvdev; 24371a86b377SEli Cohen mvdev->mdev = mdev; 24381a86b377SEli Cohen init_mvqs(ndev); 24391a86b377SEli Cohen mutex_init(&ndev->reslock); 24401a86b377SEli Cohen config = &ndev->config; 2441*246fd1caSEli Cohen err = query_mtu(mdev, &mtu); 24421a86b377SEli Cohen if (err) 24431a86b377SEli Cohen goto err_mtu; 24441a86b377SEli Cohen 2445*246fd1caSEli Cohen ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, mtu); 2446*246fd1caSEli Cohen ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 2447*246fd1caSEli Cohen 24481a86b377SEli Cohen err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac); 24491a86b377SEli Cohen if (err) 24501a86b377SEli Cohen goto err_mtu; 24511a86b377SEli Cohen 24527c9f131fSEli Cohen if (!is_zero_ether_addr(config->mac)) { 24537c9f131fSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev)); 24547c9f131fSEli Cohen err = mlx5_mpfs_add_mac(pfmdev, config->mac); 24557c9f131fSEli Cohen if (err) 24567c9f131fSEli Cohen goto err_mtu; 24575262912eSEli Cohen 24585262912eSEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MAC); 24597c9f131fSEli Cohen } 24607c9f131fSEli Cohen 24615262912eSEli Cohen config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, mlx5_vdpa_max_qps(max_vqs)); 24627d23dcdfSEli Cohen mvdev->vdev.dma_dev = &mdev->pdev->dev; 24631a86b377SEli Cohen err = mlx5_vdpa_alloc_resources(&ndev->mvdev); 24641a86b377SEli Cohen if (err) 24657c9f131fSEli Cohen goto err_mpfs; 24661a86b377SEli Cohen 24676f5312f8SEli Cohen if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 24686f5312f8SEli Cohen err = mlx5_vdpa_create_mr(mvdev, NULL); 24691a86b377SEli Cohen if (err) 24701a86b377SEli Cohen goto err_res; 24716f5312f8SEli Cohen } 24726f5312f8SEli Cohen 24736f5312f8SEli Cohen err = alloc_resources(ndev); 24746f5312f8SEli Cohen if (err) 24756f5312f8SEli Cohen goto err_mr; 24761a86b377SEli Cohen 24775262912eSEli Cohen mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_ctrl_wq"); 24785262912eSEli Cohen if (!mvdev->wq) { 24795262912eSEli Cohen err = -ENOMEM; 24805262912eSEli Cohen goto err_res2; 24815262912eSEli Cohen } 24825262912eSEli Cohen 24835262912eSEli Cohen ndev->cur_num_vqs = 2 * mlx5_vdpa_max_qps(max_vqs); 248458926c8aSEli Cohen mvdev->vdev.mdev = &mgtdev->mgtdev; 24855262912eSEli Cohen err = _vdpa_register_device(&mvdev->vdev, ndev->cur_num_vqs + 1); 24861a86b377SEli Cohen if (err) 24871a86b377SEli Cohen goto err_reg; 24881a86b377SEli Cohen 248958926c8aSEli Cohen mgtdev->ndev = ndev; 249074c9729dSLeon Romanovsky return 0; 24911a86b377SEli Cohen 24921a86b377SEli Cohen err_reg: 24935262912eSEli Cohen destroy_workqueue(mvdev->wq); 24945262912eSEli Cohen err_res2: 24951a86b377SEli Cohen free_resources(ndev); 24966f5312f8SEli Cohen err_mr: 24976f5312f8SEli Cohen mlx5_vdpa_destroy_mr(mvdev); 24981a86b377SEli Cohen err_res: 24991a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 25007c9f131fSEli Cohen err_mpfs: 25017c9f131fSEli Cohen if (!is_zero_ether_addr(config->mac)) 25027c9f131fSEli Cohen mlx5_mpfs_del_mac(pfmdev, config->mac); 25031a86b377SEli Cohen err_mtu: 25041a86b377SEli Cohen mutex_destroy(&ndev->reslock); 25051a86b377SEli Cohen put_device(&mvdev->vdev.dev); 250674c9729dSLeon Romanovsky return err; 25071a86b377SEli Cohen } 25081a86b377SEli Cohen 250958926c8aSEli Cohen static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev) 251058926c8aSEli Cohen { 251158926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 25125262912eSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(dev); 251358926c8aSEli Cohen 25145262912eSEli Cohen destroy_workqueue(mvdev->wq); 251558926c8aSEli Cohen _vdpa_unregister_device(dev); 251658926c8aSEli Cohen mgtdev->ndev = NULL; 251758926c8aSEli Cohen } 251858926c8aSEli Cohen 251958926c8aSEli Cohen static const struct vdpa_mgmtdev_ops mdev_ops = { 252058926c8aSEli Cohen .dev_add = mlx5_vdpa_dev_add, 252158926c8aSEli Cohen .dev_del = mlx5_vdpa_dev_del, 252258926c8aSEli Cohen }; 252358926c8aSEli Cohen 252458926c8aSEli Cohen static struct virtio_device_id id_table[] = { 252558926c8aSEli Cohen { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 252658926c8aSEli Cohen { 0 }, 252758926c8aSEli Cohen }; 252858926c8aSEli Cohen 252958926c8aSEli Cohen static int mlx5v_probe(struct auxiliary_device *adev, 253058926c8aSEli Cohen const struct auxiliary_device_id *id) 253158926c8aSEli Cohen 253258926c8aSEli Cohen { 253358926c8aSEli Cohen struct mlx5_adev *madev = container_of(adev, struct mlx5_adev, adev); 253458926c8aSEli Cohen struct mlx5_core_dev *mdev = madev->mdev; 253558926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev; 253658926c8aSEli Cohen int err; 253758926c8aSEli Cohen 253858926c8aSEli Cohen mgtdev = kzalloc(sizeof(*mgtdev), GFP_KERNEL); 253958926c8aSEli Cohen if (!mgtdev) 254058926c8aSEli Cohen return -ENOMEM; 254158926c8aSEli Cohen 254258926c8aSEli Cohen mgtdev->mgtdev.ops = &mdev_ops; 254358926c8aSEli Cohen mgtdev->mgtdev.device = mdev->device; 254458926c8aSEli Cohen mgtdev->mgtdev.id_table = id_table; 254558926c8aSEli Cohen mgtdev->madev = madev; 254658926c8aSEli Cohen 254758926c8aSEli Cohen err = vdpa_mgmtdev_register(&mgtdev->mgtdev); 254858926c8aSEli Cohen if (err) 254958926c8aSEli Cohen goto reg_err; 255058926c8aSEli Cohen 255158926c8aSEli Cohen dev_set_drvdata(&adev->dev, mgtdev); 255258926c8aSEli Cohen 255358926c8aSEli Cohen return 0; 255458926c8aSEli Cohen 255558926c8aSEli Cohen reg_err: 255658926c8aSEli Cohen kfree(mgtdev); 255758926c8aSEli Cohen return err; 255858926c8aSEli Cohen } 255958926c8aSEli Cohen 256074c9729dSLeon Romanovsky static void mlx5v_remove(struct auxiliary_device *adev) 25611a86b377SEli Cohen { 256258926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev; 256374c9729dSLeon Romanovsky 256458926c8aSEli Cohen mgtdev = dev_get_drvdata(&adev->dev); 256558926c8aSEli Cohen vdpa_mgmtdev_unregister(&mgtdev->mgtdev); 256658926c8aSEli Cohen kfree(mgtdev); 25671a86b377SEli Cohen } 256874c9729dSLeon Romanovsky 256974c9729dSLeon Romanovsky static const struct auxiliary_device_id mlx5v_id_table[] = { 257074c9729dSLeon Romanovsky { .name = MLX5_ADEV_NAME ".vnet", }, 257174c9729dSLeon Romanovsky {}, 257274c9729dSLeon Romanovsky }; 257374c9729dSLeon Romanovsky 257474c9729dSLeon Romanovsky MODULE_DEVICE_TABLE(auxiliary, mlx5v_id_table); 257574c9729dSLeon Romanovsky 257674c9729dSLeon Romanovsky static struct auxiliary_driver mlx5v_driver = { 257774c9729dSLeon Romanovsky .name = "vnet", 257874c9729dSLeon Romanovsky .probe = mlx5v_probe, 257974c9729dSLeon Romanovsky .remove = mlx5v_remove, 258074c9729dSLeon Romanovsky .id_table = mlx5v_id_table, 258174c9729dSLeon Romanovsky }; 258274c9729dSLeon Romanovsky 258374c9729dSLeon Romanovsky module_auxiliary_driver(mlx5v_driver); 2584