11a86b377SEli Cohen // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 21a86b377SEli Cohen /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 31a86b377SEli Cohen 474c9729dSLeon Romanovsky #include <linux/module.h> 51a86b377SEli Cohen #include <linux/vdpa.h> 674c9729dSLeon Romanovsky #include <linux/vringh.h> 774c9729dSLeon Romanovsky #include <uapi/linux/virtio_net.h> 81a86b377SEli Cohen #include <uapi/linux/virtio_ids.h> 9a007d940SEli Cohen #include <uapi/linux/vdpa.h> 101a86b377SEli Cohen #include <linux/virtio_config.h> 1174c9729dSLeon Romanovsky #include <linux/auxiliary_bus.h> 1274c9729dSLeon Romanovsky #include <linux/mlx5/cq.h> 131a86b377SEli Cohen #include <linux/mlx5/qp.h> 141a86b377SEli Cohen #include <linux/mlx5/device.h> 1574c9729dSLeon Romanovsky #include <linux/mlx5/driver.h> 161a86b377SEli Cohen #include <linux/mlx5/vport.h> 171a86b377SEli Cohen #include <linux/mlx5/fs.h> 180aae392bSLeon Romanovsky #include <linux/mlx5/mlx5_ifc_vdpa.h> 197c9f131fSEli Cohen #include <linux/mlx5/mpfs.h> 201a86b377SEli Cohen #include "mlx5_vdpa.h" 211a86b377SEli Cohen 2274c9729dSLeon Romanovsky MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 2374c9729dSLeon Romanovsky MODULE_DESCRIPTION("Mellanox VDPA driver"); 2474c9729dSLeon Romanovsky MODULE_LICENSE("Dual BSD/GPL"); 2574c9729dSLeon Romanovsky 2674c9729dSLeon Romanovsky #define to_mlx5_vdpa_ndev(__mvdev) \ 2774c9729dSLeon Romanovsky container_of(__mvdev, struct mlx5_vdpa_net, mvdev) 281a86b377SEli Cohen #define to_mvdev(__vdev) container_of((__vdev), struct mlx5_vdpa_dev, vdev) 291a86b377SEli Cohen 301a86b377SEli Cohen #define VALID_FEATURES_MASK \ 31cbb52359SNathan Chancellor (BIT_ULL(VIRTIO_NET_F_CSUM) | BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | \ 32cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_MAC) | \ 33cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | \ 34cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | \ 35cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | BIT_ULL(VIRTIO_NET_F_HOST_ECN) | BIT_ULL(VIRTIO_NET_F_HOST_UFO) | \ 36cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | BIT_ULL(VIRTIO_NET_F_STATUS) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | \ 37cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_RX) | BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | \ 38cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE) | \ 39cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | \ 40cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_RSS) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY) | \ 41cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX) | BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY) | \ 42cbb52359SNathan Chancellor BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | \ 43cbb52359SNathan Chancellor BIT_ULL(VIRTIO_F_RING_PACKED) | BIT_ULL(VIRTIO_F_ORDER_PLATFORM) | BIT_ULL(VIRTIO_F_SR_IOV)) 441a86b377SEli Cohen 451a86b377SEli Cohen #define VALID_STATUS_MASK \ 461a86b377SEli Cohen (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \ 471a86b377SEli Cohen VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED) 481a86b377SEli Cohen 49e4fc6650SEli Cohen #define MLX5_FEATURE(_mvdev, _feature) (!!((_mvdev)->actual_features & BIT_ULL(_feature))) 50e4fc6650SEli Cohen 51*baf2ad3fSEli Cohen #define MLX5V_UNTAGGED 0x1000 52*baf2ad3fSEli Cohen 531a86b377SEli Cohen struct mlx5_vdpa_net_resources { 541a86b377SEli Cohen u32 tisn; 551a86b377SEli Cohen u32 tdn; 561a86b377SEli Cohen u32 tirn; 571a86b377SEli Cohen u32 rqtn; 581a86b377SEli Cohen bool valid; 591a86b377SEli Cohen }; 601a86b377SEli Cohen 611a86b377SEli Cohen struct mlx5_vdpa_cq_buf { 621a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 631a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 641a86b377SEli Cohen int cqe_size; 651a86b377SEli Cohen int nent; 661a86b377SEli Cohen }; 671a86b377SEli Cohen 681a86b377SEli Cohen struct mlx5_vdpa_cq { 691a86b377SEli Cohen struct mlx5_core_cq mcq; 701a86b377SEli Cohen struct mlx5_vdpa_cq_buf buf; 711a86b377SEli Cohen struct mlx5_db db; 721a86b377SEli Cohen int cqe; 731a86b377SEli Cohen }; 741a86b377SEli Cohen 751a86b377SEli Cohen struct mlx5_vdpa_umem { 761a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 771a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 781a86b377SEli Cohen int size; 791a86b377SEli Cohen u32 id; 801a86b377SEli Cohen }; 811a86b377SEli Cohen 821a86b377SEli Cohen struct mlx5_vdpa_qp { 831a86b377SEli Cohen struct mlx5_core_qp mqp; 841a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 851a86b377SEli Cohen struct mlx5_db db; 861a86b377SEli Cohen u16 head; 871a86b377SEli Cohen bool fw; 881a86b377SEli Cohen }; 891a86b377SEli Cohen 901a86b377SEli Cohen struct mlx5_vq_restore_info { 911a86b377SEli Cohen u32 num_ent; 921a86b377SEli Cohen u64 desc_addr; 931a86b377SEli Cohen u64 device_addr; 941a86b377SEli Cohen u64 driver_addr; 951a86b377SEli Cohen u16 avail_index; 96b35ccebeSEli Cohen u16 used_index; 971a86b377SEli Cohen bool ready; 981a86b377SEli Cohen bool restore; 991a86b377SEli Cohen }; 1001a86b377SEli Cohen 1011a86b377SEli Cohen struct mlx5_vdpa_virtqueue { 1021a86b377SEli Cohen bool ready; 1031a86b377SEli Cohen u64 desc_addr; 1041a86b377SEli Cohen u64 device_addr; 1051a86b377SEli Cohen u64 driver_addr; 1061a86b377SEli Cohen u32 num_ent; 1071a86b377SEli Cohen 1081a86b377SEli Cohen /* Resources for implementing the notification channel from the device 1091a86b377SEli Cohen * to the driver. fwqp is the firmware end of an RC connection; the 1101a86b377SEli Cohen * other end is vqqp used by the driver. cq is is where completions are 1111a86b377SEli Cohen * reported. 1121a86b377SEli Cohen */ 1131a86b377SEli Cohen struct mlx5_vdpa_cq cq; 1141a86b377SEli Cohen struct mlx5_vdpa_qp fwqp; 1151a86b377SEli Cohen struct mlx5_vdpa_qp vqqp; 1161a86b377SEli Cohen 1171a86b377SEli Cohen /* umem resources are required for the virtqueue operation. They're use 1181a86b377SEli Cohen * is internal and they must be provided by the driver. 1191a86b377SEli Cohen */ 1201a86b377SEli Cohen struct mlx5_vdpa_umem umem1; 1211a86b377SEli Cohen struct mlx5_vdpa_umem umem2; 1221a86b377SEli Cohen struct mlx5_vdpa_umem umem3; 1231a86b377SEli Cohen 1241892a3d4SEli Cohen u32 counter_set_id; 1251a86b377SEli Cohen bool initialized; 1261a86b377SEli Cohen int index; 1271a86b377SEli Cohen u32 virtq_id; 1281a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 1291a86b377SEli Cohen u16 avail_idx; 130b35ccebeSEli Cohen u16 used_idx; 1311a86b377SEli Cohen int fw_state; 1321a86b377SEli Cohen 1331a86b377SEli Cohen /* keep last in the struct */ 1341a86b377SEli Cohen struct mlx5_vq_restore_info ri; 1351a86b377SEli Cohen }; 1361a86b377SEli Cohen 137e4fc6650SEli Cohen static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx) 138e4fc6650SEli Cohen { 139f8ae3a48SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) { 140f8ae3a48SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 141f8ae3a48SEli Cohen return idx < 2; 142f8ae3a48SEli Cohen else 143f8ae3a48SEli Cohen return idx < 3; 144f8ae3a48SEli Cohen } 145e4fc6650SEli Cohen 146f8ae3a48SEli Cohen return idx <= mvdev->max_idx; 147e4fc6650SEli Cohen } 148e4fc6650SEli Cohen 149*baf2ad3fSEli Cohen #define MLX5V_MACVLAN_SIZE 256 150*baf2ad3fSEli Cohen 1511a86b377SEli Cohen struct mlx5_vdpa_net { 1521a86b377SEli Cohen struct mlx5_vdpa_dev mvdev; 1531a86b377SEli Cohen struct mlx5_vdpa_net_resources res; 1541a86b377SEli Cohen struct virtio_net_config config; 15575560522SEli Cohen struct mlx5_vdpa_virtqueue *vqs; 15675560522SEli Cohen struct vdpa_callback *event_cbs; 1571a86b377SEli Cohen 1581a86b377SEli Cohen /* Serialize vq resources creation and destruction. This is required 1591a86b377SEli Cohen * since memory map might change and we need to destroy and create 1601a86b377SEli Cohen * resources while driver in operational. 1611a86b377SEli Cohen */ 162759ae7f9SEli Cohen struct rw_semaphore reslock; 1631a86b377SEli Cohen struct mlx5_flow_table *rxft; 1641a86b377SEli Cohen bool setup; 1655262912eSEli Cohen u32 cur_num_vqs; 166acde3929SEli Cohen u32 rqt_size; 167edf747afSEli Cohen struct notifier_block nb; 168edf747afSEli Cohen struct vdpa_callback config_cb; 16955ebf0d6SJason Wang struct mlx5_vdpa_wq_ent cvq_ent; 170*baf2ad3fSEli Cohen struct hlist_head macvlan_hash[MLX5V_MACVLAN_SIZE]; 171*baf2ad3fSEli Cohen }; 172*baf2ad3fSEli Cohen 173*baf2ad3fSEli Cohen struct macvlan_node { 174*baf2ad3fSEli Cohen struct hlist_node hlist; 175*baf2ad3fSEli Cohen struct mlx5_flow_handle *ucast_rule; 176*baf2ad3fSEli Cohen struct mlx5_flow_handle *mcast_rule; 177*baf2ad3fSEli Cohen u64 macvlan; 1781a86b377SEli Cohen }; 1791a86b377SEli Cohen 1801a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev); 1811a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev); 182ae0428deSEli Cohen static int setup_driver(struct mlx5_vdpa_dev *mvdev); 1831a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev); 1841a86b377SEli Cohen 1851a86b377SEli Cohen static bool mlx5_vdpa_debug; 1861a86b377SEli Cohen 1875262912eSEli Cohen #define MLX5_CVQ_MAX_ENT 16 1885262912eSEli Cohen 1891a86b377SEli Cohen #define MLX5_LOG_VIO_FLAG(_feature) \ 1901a86b377SEli Cohen do { \ 191cbb52359SNathan Chancellor if (features & BIT_ULL(_feature)) \ 1921a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_feature); \ 1931a86b377SEli Cohen } while (0) 1941a86b377SEli Cohen 1951a86b377SEli Cohen #define MLX5_LOG_VIO_STAT(_status) \ 1961a86b377SEli Cohen do { \ 1971a86b377SEli Cohen if (status & (_status)) \ 1981a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_status); \ 1991a86b377SEli Cohen } while (0) 2001a86b377SEli Cohen 20152893733SEli Cohen /* TODO: cross-endian support */ 20252893733SEli Cohen static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev) 20352893733SEli Cohen { 20452893733SEli Cohen return virtio_legacy_is_little_endian() || 20552893733SEli Cohen (mvdev->actual_features & BIT_ULL(VIRTIO_F_VERSION_1)); 20652893733SEli Cohen } 20752893733SEli Cohen 20852893733SEli Cohen static u16 mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev *mvdev, __virtio16 val) 20952893733SEli Cohen { 21052893733SEli Cohen return __virtio16_to_cpu(mlx5_vdpa_is_little_endian(mvdev), val); 21152893733SEli Cohen } 21252893733SEli Cohen 21352893733SEli Cohen static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val) 21452893733SEli Cohen { 21552893733SEli Cohen return __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), val); 21652893733SEli Cohen } 21752893733SEli Cohen 2185262912eSEli Cohen static u16 ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev) 2195262912eSEli Cohen { 22052893733SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) 22152893733SEli Cohen return 2; 22252893733SEli Cohen 223acde3929SEli Cohen return mvdev->max_vqs; 2245262912eSEli Cohen } 2255262912eSEli Cohen 2265262912eSEli Cohen static bool is_ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev, u16 idx) 2275262912eSEli Cohen { 2285262912eSEli Cohen return idx == ctrl_vq_idx(mvdev); 2295262912eSEli Cohen } 2305262912eSEli Cohen 2311a86b377SEli Cohen static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set) 2321a86b377SEli Cohen { 2331a86b377SEli Cohen if (status & ~VALID_STATUS_MASK) 2341a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n", 2351a86b377SEli Cohen status & ~VALID_STATUS_MASK); 2361a86b377SEli Cohen 2371a86b377SEli Cohen if (!mlx5_vdpa_debug) 2381a86b377SEli Cohen return; 2391a86b377SEli Cohen 2401a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get"); 2411a86b377SEli Cohen if (set && !status) { 2421a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver resets the device\n"); 2431a86b377SEli Cohen return; 2441a86b377SEli Cohen } 2451a86b377SEli Cohen 2461a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE); 2471a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER); 2481a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK); 2491a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK); 2501a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET); 2511a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED); 2521a86b377SEli Cohen } 2531a86b377SEli Cohen 2541a86b377SEli Cohen static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set) 2551a86b377SEli Cohen { 2561a86b377SEli Cohen if (features & ~VALID_FEATURES_MASK) 2571a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n", 2581a86b377SEli Cohen features & ~VALID_FEATURES_MASK); 2591a86b377SEli Cohen 2601a86b377SEli Cohen if (!mlx5_vdpa_debug) 2611a86b377SEli Cohen return; 2621a86b377SEli Cohen 2631a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads"); 2641a86b377SEli Cohen if (!features) 2651a86b377SEli Cohen mlx5_vdpa_info(mvdev, "all feature bits are cleared\n"); 2661a86b377SEli Cohen 2671a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM); 2681a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM); 2691a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 2701a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU); 2711a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC); 2721a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4); 2731a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6); 2741a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN); 2751a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO); 2761a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4); 2771a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6); 2781a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN); 2791a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO); 2801a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF); 2811a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS); 2821a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ); 2831a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX); 2841a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN); 2851a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA); 2861a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE); 2871a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ); 2881a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR); 2891a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT); 2901a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS); 2911a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT); 2921a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY); 2931a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX); 2941a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY); 2951a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT); 2961a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1); 2971a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM); 2981a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED); 2991a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM); 3001a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV); 3011a86b377SEli Cohen } 3021a86b377SEli Cohen 3031a86b377SEli Cohen static int create_tis(struct mlx5_vdpa_net *ndev) 3041a86b377SEli Cohen { 3051a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; 3061a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 3071a86b377SEli Cohen void *tisc; 3081a86b377SEli Cohen int err; 3091a86b377SEli Cohen 3101a86b377SEli Cohen tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3111a86b377SEli Cohen MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); 3121a86b377SEli Cohen err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); 3131a86b377SEli Cohen if (err) 3141a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err); 3151a86b377SEli Cohen 3161a86b377SEli Cohen return err; 3171a86b377SEli Cohen } 3181a86b377SEli Cohen 3191a86b377SEli Cohen static void destroy_tis(struct mlx5_vdpa_net *ndev) 3201a86b377SEli Cohen { 3211a86b377SEli Cohen mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); 3221a86b377SEli Cohen } 3231a86b377SEli Cohen 3241a86b377SEli Cohen #define MLX5_VDPA_CQE_SIZE 64 3251a86b377SEli Cohen #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE) 3261a86b377SEli Cohen 3271a86b377SEli Cohen static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) 3281a86b377SEli Cohen { 3291a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &buf->frag_buf; 3301a86b377SEli Cohen u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE; 3311a86b377SEli Cohen u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE; 3321a86b377SEli Cohen int err; 3331a86b377SEli Cohen 3341a86b377SEli Cohen err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, 3351a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3361a86b377SEli Cohen if (err) 3371a86b377SEli Cohen return err; 3381a86b377SEli Cohen 3391a86b377SEli Cohen mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc); 3401a86b377SEli Cohen 3411a86b377SEli Cohen buf->cqe_size = MLX5_VDPA_CQE_SIZE; 3421a86b377SEli Cohen buf->nent = nent; 3431a86b377SEli Cohen 3441a86b377SEli Cohen return 0; 3451a86b377SEli Cohen } 3461a86b377SEli Cohen 3471a86b377SEli Cohen static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) 3481a86b377SEli Cohen { 3491a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &umem->frag_buf; 3501a86b377SEli Cohen 3511a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, 3521a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3531a86b377SEli Cohen } 3541a86b377SEli Cohen 3551a86b377SEli Cohen static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) 3561a86b377SEli Cohen { 3571a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); 3581a86b377SEli Cohen } 3591a86b377SEli Cohen 3601a86b377SEli Cohen static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n) 3611a86b377SEli Cohen { 3621a86b377SEli Cohen return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n); 3631a86b377SEli Cohen } 3641a86b377SEli Cohen 3651a86b377SEli Cohen static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf) 3661a86b377SEli Cohen { 3671a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 3681a86b377SEli Cohen void *cqe; 3691a86b377SEli Cohen int i; 3701a86b377SEli Cohen 3711a86b377SEli Cohen for (i = 0; i < buf->nent; i++) { 3721a86b377SEli Cohen cqe = get_cqe(vcq, i); 3731a86b377SEli Cohen cqe64 = cqe; 3741a86b377SEli Cohen cqe64->op_own = MLX5_CQE_INVALID << 4; 3751a86b377SEli Cohen } 3761a86b377SEli Cohen } 3771a86b377SEli Cohen 3781a86b377SEli Cohen static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n) 3791a86b377SEli Cohen { 3801a86b377SEli Cohen struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1)); 3811a86b377SEli Cohen 3821a86b377SEli Cohen if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && 3831a86b377SEli Cohen !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe))) 3841a86b377SEli Cohen return cqe64; 3851a86b377SEli Cohen 3861a86b377SEli Cohen return NULL; 3871a86b377SEli Cohen } 3881a86b377SEli Cohen 3891a86b377SEli Cohen static void rx_post(struct mlx5_vdpa_qp *vqp, int n) 3901a86b377SEli Cohen { 3911a86b377SEli Cohen vqp->head += n; 3921a86b377SEli Cohen vqp->db.db[0] = cpu_to_be32(vqp->head); 3931a86b377SEli Cohen } 3941a86b377SEli Cohen 3951a86b377SEli Cohen static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, 3961a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq, u32 num_ent) 3971a86b377SEli Cohen { 3981a86b377SEli Cohen struct mlx5_vdpa_qp *vqp; 3991a86b377SEli Cohen __be64 *pas; 4001a86b377SEli Cohen void *qpc; 4011a86b377SEli Cohen 4021a86b377SEli Cohen vqp = fw ? &mvq->fwqp : &mvq->vqqp; 4031a86b377SEli Cohen MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); 4041a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4051a86b377SEli Cohen if (vqp->fw) { 4061a86b377SEli Cohen /* Firmware QP is allocated by the driver for the firmware's 4071a86b377SEli Cohen * use so we can skip part of the params as they will be chosen by firmware 4081a86b377SEli Cohen */ 4091a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4101a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ); 4111a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 4121a86b377SEli Cohen return; 4131a86b377SEli Cohen } 4141a86b377SEli Cohen 4151a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4161a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4171a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4181a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4191a86b377SEli Cohen MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); 4201a86b377SEli Cohen MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 4211a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 4221a86b377SEli Cohen MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn); 4231a86b377SEli Cohen MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent)); 4241a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 4251a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas); 4261a86b377SEli Cohen mlx5_fill_page_frag_array(&vqp->frag_buf, pas); 4271a86b377SEli Cohen } 4281a86b377SEli Cohen 4291a86b377SEli Cohen static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) 4301a86b377SEli Cohen { 4311a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, 4321a86b377SEli Cohen num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf, 4331a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 4341a86b377SEli Cohen } 4351a86b377SEli Cohen 4361a86b377SEli Cohen static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 4371a86b377SEli Cohen { 4381a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); 4391a86b377SEli Cohen } 4401a86b377SEli Cohen 4411a86b377SEli Cohen static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 4421a86b377SEli Cohen struct mlx5_vdpa_qp *vqp) 4431a86b377SEli Cohen { 4441a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 4451a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_qp_in); 4461a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 4471a86b377SEli Cohen void *qpc; 4481a86b377SEli Cohen void *in; 4491a86b377SEli Cohen int err; 4501a86b377SEli Cohen 4511a86b377SEli Cohen if (!vqp->fw) { 4521a86b377SEli Cohen vqp = &mvq->vqqp; 4531a86b377SEli Cohen err = rq_buf_alloc(ndev, vqp, mvq->num_ent); 4541a86b377SEli Cohen if (err) 4551a86b377SEli Cohen return err; 4561a86b377SEli Cohen 4571a86b377SEli Cohen err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); 4581a86b377SEli Cohen if (err) 4591a86b377SEli Cohen goto err_db; 4601a86b377SEli Cohen inlen += vqp->frag_buf.npages * sizeof(__be64); 4611a86b377SEli Cohen } 4621a86b377SEli Cohen 4631a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 4641a86b377SEli Cohen if (!in) { 4651a86b377SEli Cohen err = -ENOMEM; 4661a86b377SEli Cohen goto err_kzalloc; 4671a86b377SEli Cohen } 4681a86b377SEli Cohen 4691a86b377SEli Cohen qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); 4701a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4711a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4721a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4731a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4741a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4751a86b377SEli Cohen if (!vqp->fw) 4761a86b377SEli Cohen MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma); 4771a86b377SEli Cohen MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 4781a86b377SEli Cohen err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 4791a86b377SEli Cohen kfree(in); 4801a86b377SEli Cohen if (err) 4811a86b377SEli Cohen goto err_kzalloc; 4821a86b377SEli Cohen 4831a86b377SEli Cohen vqp->mqp.uid = ndev->mvdev.res.uid; 4841a86b377SEli Cohen vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn); 4851a86b377SEli Cohen 4861a86b377SEli Cohen if (!vqp->fw) 4871a86b377SEli Cohen rx_post(vqp, mvq->num_ent); 4881a86b377SEli Cohen 4891a86b377SEli Cohen return 0; 4901a86b377SEli Cohen 4911a86b377SEli Cohen err_kzalloc: 4921a86b377SEli Cohen if (!vqp->fw) 4931a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 4941a86b377SEli Cohen err_db: 4951a86b377SEli Cohen if (!vqp->fw) 4961a86b377SEli Cohen rq_buf_free(ndev, vqp); 4971a86b377SEli Cohen 4981a86b377SEli Cohen return err; 4991a86b377SEli Cohen } 5001a86b377SEli Cohen 5011a86b377SEli Cohen static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 5021a86b377SEli Cohen { 5031a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 5041a86b377SEli Cohen 5051a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 5061a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn); 5071a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); 5081a86b377SEli Cohen if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) 5091a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); 5101a86b377SEli Cohen if (!vqp->fw) { 5111a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 5121a86b377SEli Cohen rq_buf_free(ndev, vqp); 5131a86b377SEli Cohen } 5141a86b377SEli Cohen } 5151a86b377SEli Cohen 5161a86b377SEli Cohen static void *next_cqe_sw(struct mlx5_vdpa_cq *cq) 5171a86b377SEli Cohen { 5181a86b377SEli Cohen return get_sw_cqe(cq, cq->mcq.cons_index); 5191a86b377SEli Cohen } 5201a86b377SEli Cohen 5211a86b377SEli Cohen static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq) 5221a86b377SEli Cohen { 5231a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 5241a86b377SEli Cohen 5251a86b377SEli Cohen cqe64 = next_cqe_sw(vcq); 5261a86b377SEli Cohen if (!cqe64) 5271a86b377SEli Cohen return -EAGAIN; 5281a86b377SEli Cohen 5291a86b377SEli Cohen vcq->mcq.cons_index++; 5301a86b377SEli Cohen return 0; 5311a86b377SEli Cohen } 5321a86b377SEli Cohen 5331a86b377SEli Cohen static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num) 5341a86b377SEli Cohen { 535db296d25SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 536db296d25SEli Cohen struct vdpa_callback *event_cb; 537db296d25SEli Cohen 538db296d25SEli Cohen event_cb = &ndev->event_cbs[mvq->index]; 5391a86b377SEli Cohen mlx5_cq_set_ci(&mvq->cq.mcq); 54083ef73b2SEli Cohen 54183ef73b2SEli Cohen /* make sure CQ cosumer update is visible to the hardware before updating 54283ef73b2SEli Cohen * RX doorbell record. 54383ef73b2SEli Cohen */ 54483ef73b2SEli Cohen dma_wmb(); 5451a86b377SEli Cohen rx_post(&mvq->vqqp, num); 546db296d25SEli Cohen if (event_cb->callback) 547db296d25SEli Cohen event_cb->callback(event_cb->private); 5481a86b377SEli Cohen } 5491a86b377SEli Cohen 5501a86b377SEli Cohen static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) 5511a86b377SEli Cohen { 5521a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq); 5531a86b377SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 5541a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5551a86b377SEli Cohen int num = 0; 5561a86b377SEli Cohen 5571a86b377SEli Cohen while (!mlx5_vdpa_poll_one(&mvq->cq)) { 5581a86b377SEli Cohen num++; 5591a86b377SEli Cohen if (num > mvq->num_ent / 2) { 5601a86b377SEli Cohen /* If completions keep coming while we poll, we want to 5611a86b377SEli Cohen * let the hardware know that we consumed them by 5621a86b377SEli Cohen * updating the doorbell record. We also let vdpa core 5631a86b377SEli Cohen * know about this so it passes it on the virtio driver 5641a86b377SEli Cohen * on the guest. 5651a86b377SEli Cohen */ 5661a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 5671a86b377SEli Cohen num = 0; 5681a86b377SEli Cohen } 5691a86b377SEli Cohen } 5701a86b377SEli Cohen 5711a86b377SEli Cohen if (num) 5721a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 5731a86b377SEli Cohen 5741a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 5751a86b377SEli Cohen } 5761a86b377SEli Cohen 5771a86b377SEli Cohen static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) 5781a86b377SEli Cohen { 5791a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 5801a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 5811a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5821a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 5831a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 5841a86b377SEli Cohen __be64 *pas; 5851a86b377SEli Cohen int inlen; 5861a86b377SEli Cohen void *cqc; 5871a86b377SEli Cohen void *in; 5881a86b377SEli Cohen int err; 5891a86b377SEli Cohen int eqn; 5901a86b377SEli Cohen 5911a86b377SEli Cohen err = mlx5_db_alloc(mdev, &vcq->db); 5921a86b377SEli Cohen if (err) 5931a86b377SEli Cohen return err; 5941a86b377SEli Cohen 5951a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 5961a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 5971a86b377SEli Cohen vcq->mcq.cqe_sz = 64; 5981a86b377SEli Cohen 5991a86b377SEli Cohen err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); 6001a86b377SEli Cohen if (err) 6011a86b377SEli Cohen goto err_db; 6021a86b377SEli Cohen 6031a86b377SEli Cohen cq_frag_buf_init(vcq, &vcq->buf); 6041a86b377SEli Cohen 6051a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 6061a86b377SEli Cohen MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages; 6071a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 6081a86b377SEli Cohen if (!in) { 6091a86b377SEli Cohen err = -ENOMEM; 6101a86b377SEli Cohen goto err_vzalloc; 6111a86b377SEli Cohen } 6121a86b377SEli Cohen 6131a86b377SEli Cohen MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); 6141a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 6151a86b377SEli Cohen mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas); 6161a86b377SEli Cohen 6171a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 6181a86b377SEli Cohen MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 6191a86b377SEli Cohen 6201a86b377SEli Cohen /* Use vector 0 by default. Consider adding code to choose least used 6211a86b377SEli Cohen * vector. 6221a86b377SEli Cohen */ 623563476aeSShay Drory err = mlx5_vector2eqn(mdev, 0, &eqn); 6241a86b377SEli Cohen if (err) 6251a86b377SEli Cohen goto err_vec; 6261a86b377SEli Cohen 6271a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 6281a86b377SEli Cohen MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent)); 6291a86b377SEli Cohen MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); 630616d5769STal Gilboa MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 6311a86b377SEli Cohen MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma); 6321a86b377SEli Cohen 6331a86b377SEli Cohen err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out)); 6341a86b377SEli Cohen if (err) 6351a86b377SEli Cohen goto err_vec; 6361a86b377SEli Cohen 6371a86b377SEli Cohen vcq->mcq.comp = mlx5_vdpa_cq_comp; 6381a86b377SEli Cohen vcq->cqe = num_ent; 6391a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 6401a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 6411a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 6421a86b377SEli Cohen kfree(in); 6431a86b377SEli Cohen return 0; 6441a86b377SEli Cohen 6451a86b377SEli Cohen err_vec: 6461a86b377SEli Cohen kfree(in); 6471a86b377SEli Cohen err_vzalloc: 6481a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 6491a86b377SEli Cohen err_db: 6501a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 6511a86b377SEli Cohen return err; 6521a86b377SEli Cohen } 6531a86b377SEli Cohen 6541a86b377SEli Cohen static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) 6551a86b377SEli Cohen { 6561a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 6571a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 6581a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 6591a86b377SEli Cohen 6601a86b377SEli Cohen if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) { 6611a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); 6621a86b377SEli Cohen return; 6631a86b377SEli Cohen } 6641a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 6651a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 6661a86b377SEli Cohen } 6671a86b377SEli Cohen 66871ab6a7cSEli Cohen static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, 6691a86b377SEli Cohen struct mlx5_vdpa_umem **umemp) 6701a86b377SEli Cohen { 6711a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 6721a86b377SEli Cohen int p_a; 6731a86b377SEli Cohen int p_b; 6741a86b377SEli Cohen 6751a86b377SEli Cohen switch (num) { 6761a86b377SEli Cohen case 1: 6771a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a); 6781a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b); 6791a86b377SEli Cohen *umemp = &mvq->umem1; 6801a86b377SEli Cohen break; 6811a86b377SEli Cohen case 2: 6821a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a); 6831a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b); 6841a86b377SEli Cohen *umemp = &mvq->umem2; 6851a86b377SEli Cohen break; 6861a86b377SEli Cohen case 3: 6871a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a); 6881a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b); 6891a86b377SEli Cohen *umemp = &mvq->umem3; 6901a86b377SEli Cohen break; 6911a86b377SEli Cohen } 69271ab6a7cSEli Cohen (*umemp)->size = p_a * mvq->num_ent + p_b; 6931a86b377SEli Cohen } 6941a86b377SEli Cohen 6951a86b377SEli Cohen static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) 6961a86b377SEli Cohen { 6971a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); 6981a86b377SEli Cohen } 6991a86b377SEli Cohen 7001a86b377SEli Cohen static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 7011a86b377SEli Cohen { 7021a86b377SEli Cohen int inlen; 7031a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {}; 7041a86b377SEli Cohen void *um; 7051a86b377SEli Cohen void *in; 7061a86b377SEli Cohen int err; 7071a86b377SEli Cohen __be64 *pas; 7081a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 7091a86b377SEli Cohen 71071ab6a7cSEli Cohen set_umem_size(ndev, mvq, num, &umem); 71171ab6a7cSEli Cohen err = umem_frag_buf_alloc(ndev, umem, umem->size); 7121a86b377SEli Cohen if (err) 7131a86b377SEli Cohen return err; 7141a86b377SEli Cohen 7151a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages; 7161a86b377SEli Cohen 7171a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 7181a86b377SEli Cohen if (!in) { 7191a86b377SEli Cohen err = -ENOMEM; 7201a86b377SEli Cohen goto err_in; 7211a86b377SEli Cohen } 7221a86b377SEli Cohen 7231a86b377SEli Cohen MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM); 7241a86b377SEli Cohen MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); 7251a86b377SEli Cohen um = MLX5_ADDR_OF(create_umem_in, in, umem); 7261a86b377SEli Cohen MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 7271a86b377SEli Cohen MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages); 7281a86b377SEli Cohen 7291a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]); 7301a86b377SEli Cohen mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW); 7311a86b377SEli Cohen 7321a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 7331a86b377SEli Cohen if (err) { 7341a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); 7351a86b377SEli Cohen goto err_cmd; 7361a86b377SEli Cohen } 7371a86b377SEli Cohen 7381a86b377SEli Cohen kfree(in); 7391a86b377SEli Cohen umem->id = MLX5_GET(create_umem_out, out, umem_id); 7401a86b377SEli Cohen 7411a86b377SEli Cohen return 0; 7421a86b377SEli Cohen 7431a86b377SEli Cohen err_cmd: 7441a86b377SEli Cohen kfree(in); 7451a86b377SEli Cohen err_in: 7461a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7471a86b377SEli Cohen return err; 7481a86b377SEli Cohen } 7491a86b377SEli Cohen 7501a86b377SEli Cohen static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 7511a86b377SEli Cohen { 7521a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {}; 7531a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {}; 7541a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 7551a86b377SEli Cohen 7561a86b377SEli Cohen switch (num) { 7571a86b377SEli Cohen case 1: 7581a86b377SEli Cohen umem = &mvq->umem1; 7591a86b377SEli Cohen break; 7601a86b377SEli Cohen case 2: 7611a86b377SEli Cohen umem = &mvq->umem2; 7621a86b377SEli Cohen break; 7631a86b377SEli Cohen case 3: 7641a86b377SEli Cohen umem = &mvq->umem3; 7651a86b377SEli Cohen break; 7661a86b377SEli Cohen } 7671a86b377SEli Cohen 7681a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM); 7691a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, umem_id, umem->id); 7701a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 7711a86b377SEli Cohen return; 7721a86b377SEli Cohen 7731a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7741a86b377SEli Cohen } 7751a86b377SEli Cohen 7761a86b377SEli Cohen static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7771a86b377SEli Cohen { 7781a86b377SEli Cohen int num; 7791a86b377SEli Cohen int err; 7801a86b377SEli Cohen 7811a86b377SEli Cohen for (num = 1; num <= 3; num++) { 7821a86b377SEli Cohen err = create_umem(ndev, mvq, num); 7831a86b377SEli Cohen if (err) 7841a86b377SEli Cohen goto err_umem; 7851a86b377SEli Cohen } 7861a86b377SEli Cohen return 0; 7871a86b377SEli Cohen 7881a86b377SEli Cohen err_umem: 7891a86b377SEli Cohen for (num--; num > 0; num--) 7901a86b377SEli Cohen umem_destroy(ndev, mvq, num); 7911a86b377SEli Cohen 7921a86b377SEli Cohen return err; 7931a86b377SEli Cohen } 7941a86b377SEli Cohen 7951a86b377SEli Cohen static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7961a86b377SEli Cohen { 7971a86b377SEli Cohen int num; 7981a86b377SEli Cohen 7991a86b377SEli Cohen for (num = 3; num > 0; num--) 8001a86b377SEli Cohen umem_destroy(ndev, mvq, num); 8011a86b377SEli Cohen } 8021a86b377SEli Cohen 8031a86b377SEli Cohen static int get_queue_type(struct mlx5_vdpa_net *ndev) 8041a86b377SEli Cohen { 8051a86b377SEli Cohen u32 type_mask; 8061a86b377SEli Cohen 8071a86b377SEli Cohen type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); 8081a86b377SEli Cohen 8091a86b377SEli Cohen /* prefer split queue */ 810879753c8SEli Cohen if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT) 8111a86b377SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT; 812879753c8SEli Cohen 813879753c8SEli Cohen WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED)); 814879753c8SEli Cohen 815879753c8SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED; 8161a86b377SEli Cohen } 8171a86b377SEli Cohen 8181a86b377SEli Cohen static bool vq_is_tx(u16 idx) 8191a86b377SEli Cohen { 8201a86b377SEli Cohen return idx % 2; 8211a86b377SEli Cohen } 8221a86b377SEli Cohen 8231a86b377SEli Cohen static u16 get_features_12_3(u64 features) 8241a86b377SEli Cohen { 825cbb52359SNathan Chancellor return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) | 826cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) | 827cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) | 828cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6); 8291a86b377SEli Cohen } 8301a86b377SEli Cohen 8311892a3d4SEli Cohen static bool counters_supported(const struct mlx5_vdpa_dev *mvdev) 8321892a3d4SEli Cohen { 8331892a3d4SEli Cohen return MLX5_CAP_GEN_64(mvdev->mdev, general_obj_types) & 8341892a3d4SEli Cohen BIT_ULL(MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 8351892a3d4SEli Cohen } 8361892a3d4SEli Cohen 8371a86b377SEli Cohen static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 8381a86b377SEli Cohen { 8391a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); 8401a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {}; 8411a86b377SEli Cohen void *obj_context; 8421a86b377SEli Cohen void *cmd_hdr; 8431a86b377SEli Cohen void *vq_ctx; 8441a86b377SEli Cohen void *in; 8451a86b377SEli Cohen int err; 8461a86b377SEli Cohen 8471a86b377SEli Cohen err = umems_create(ndev, mvq); 8481a86b377SEli Cohen if (err) 8491a86b377SEli Cohen return err; 8501a86b377SEli Cohen 8511a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 8521a86b377SEli Cohen if (!in) { 8531a86b377SEli Cohen err = -ENOMEM; 8541a86b377SEli Cohen goto err_alloc; 8551a86b377SEli Cohen } 8561a86b377SEli Cohen 8571a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr); 8581a86b377SEli Cohen 8591a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 8601a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 8611a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 8621a86b377SEli Cohen 8631a86b377SEli Cohen obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context); 8641a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); 865b35ccebeSEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); 8661a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, 8671a86b377SEli Cohen get_features_12_3(ndev->mvdev.actual_features)); 8681a86b377SEli Cohen vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context); 8691a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); 8701a86b377SEli Cohen 8711a86b377SEli Cohen if (vq_is_tx(mvq->index)) 8721a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); 8731a86b377SEli Cohen 8741a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 8751a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); 8761a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 8771a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); 8781a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 8794b454a82SEli Cohen !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); 8801a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); 8811a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); 8821a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); 88383fec3f1SAharon Landau MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey); 8841a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); 8851a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); 8861a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id); 887e3011776SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem2.size); 8881a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id); 889e3011776SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem3.size); 8901a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); 8911892a3d4SEli Cohen if (counters_supported(&ndev->mvdev)) 8921892a3d4SEli Cohen MLX5_SET(virtio_q, vq_ctx, counter_set_id, mvq->counter_set_id); 8931a86b377SEli Cohen 8941a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 8951a86b377SEli Cohen if (err) 8961a86b377SEli Cohen goto err_cmd; 8971a86b377SEli Cohen 8981a86b377SEli Cohen kfree(in); 8991a86b377SEli Cohen mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 9001a86b377SEli Cohen 9011a86b377SEli Cohen return 0; 9021a86b377SEli Cohen 9031a86b377SEli Cohen err_cmd: 9041a86b377SEli Cohen kfree(in); 9051a86b377SEli Cohen err_alloc: 9061a86b377SEli Cohen umems_destroy(ndev, mvq); 9071a86b377SEli Cohen return err; 9081a86b377SEli Cohen } 9091a86b377SEli Cohen 9101a86b377SEli Cohen static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 9111a86b377SEli Cohen { 9121a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {}; 9131a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {}; 9141a86b377SEli Cohen 9151a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode, 9161a86b377SEli Cohen MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 9171a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id); 9181a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); 9191a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type, 9201a86b377SEli Cohen MLX5_OBJ_TYPE_VIRTIO_NET_Q); 9211a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { 9221a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); 9231a86b377SEli Cohen return; 9241a86b377SEli Cohen } 9251a86b377SEli Cohen umems_destroy(ndev, mvq); 9261a86b377SEli Cohen } 9271a86b377SEli Cohen 9281a86b377SEli Cohen static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 9291a86b377SEli Cohen { 9301a86b377SEli Cohen return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn; 9311a86b377SEli Cohen } 9321a86b377SEli Cohen 9331a86b377SEli Cohen static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 9341a86b377SEli Cohen { 9351a86b377SEli Cohen return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn; 9361a86b377SEli Cohen } 9371a86b377SEli Cohen 9381a86b377SEli Cohen static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, 9391a86b377SEli Cohen int *outlen, u32 qpn, u32 rqpn) 9401a86b377SEli Cohen { 9411a86b377SEli Cohen void *qpc; 9421a86b377SEli Cohen void *pp; 9431a86b377SEli Cohen 9441a86b377SEli Cohen switch (cmd) { 9451a86b377SEli Cohen case MLX5_CMD_OP_2RST_QP: 9461a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in); 9471a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out); 9481a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9491a86b377SEli Cohen *out = kzalloc(*outlen, GFP_KERNEL); 950f31231bfSColin Ian King if (!*in || !*out) 9511a86b377SEli Cohen goto outerr; 9521a86b377SEli Cohen 9531a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, opcode, cmd); 9541a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); 9551a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, qpn, qpn); 9561a86b377SEli Cohen break; 9571a86b377SEli Cohen case MLX5_CMD_OP_RST2INIT_QP: 9581a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in); 9591a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out); 9601a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9611a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL); 962f31231bfSColin Ian King if (!*in || !*out) 9631a86b377SEli Cohen goto outerr; 9641a86b377SEli Cohen 9651a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, opcode, cmd); 9661a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); 9671a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, qpn, qpn); 9681a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9691a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9701a86b377SEli Cohen MLX5_SET(qpc, qpc, rwe, 1); 9711a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9721a86b377SEli Cohen MLX5_SET(ads, pp, vhca_port_num, 1); 9731a86b377SEli Cohen break; 9741a86b377SEli Cohen case MLX5_CMD_OP_INIT2RTR_QP: 9751a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in); 9761a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out); 9771a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9781a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL); 979f31231bfSColin Ian King if (!*in || !*out) 9801a86b377SEli Cohen goto outerr; 9811a86b377SEli Cohen 9821a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, opcode, cmd); 9831a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); 9841a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, qpn, qpn); 9851a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9861a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 9871a86b377SEli Cohen MLX5_SET(qpc, qpc, log_msg_max, 30); 9881a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9891a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9901a86b377SEli Cohen MLX5_SET(ads, pp, fl, 1); 9911a86b377SEli Cohen break; 9921a86b377SEli Cohen case MLX5_CMD_OP_RTR2RTS_QP: 9931a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in); 9941a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out); 9951a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9961a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL); 997f31231bfSColin Ian King if (!*in || !*out) 9981a86b377SEli Cohen goto outerr; 9991a86b377SEli Cohen 10001a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd); 10011a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); 10021a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn); 10031a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 10041a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 10051a86b377SEli Cohen MLX5_SET(ads, pp, ack_timeout, 14); 10061a86b377SEli Cohen MLX5_SET(qpc, qpc, retry_count, 7); 10071a86b377SEli Cohen MLX5_SET(qpc, qpc, rnr_retry, 7); 10081a86b377SEli Cohen break; 10091a86b377SEli Cohen default: 1010f31231bfSColin Ian King goto outerr_nullify; 10111a86b377SEli Cohen } 10121a86b377SEli Cohen 10131a86b377SEli Cohen return; 10141a86b377SEli Cohen 10151a86b377SEli Cohen outerr: 10161a86b377SEli Cohen kfree(*in); 10171a86b377SEli Cohen kfree(*out); 1018f31231bfSColin Ian King outerr_nullify: 10191a86b377SEli Cohen *in = NULL; 10201a86b377SEli Cohen *out = NULL; 10211a86b377SEli Cohen } 10221a86b377SEli Cohen 10231a86b377SEli Cohen static void free_inout(void *in, void *out) 10241a86b377SEli Cohen { 10251a86b377SEli Cohen kfree(in); 10261a86b377SEli Cohen kfree(out); 10271a86b377SEli Cohen } 10281a86b377SEli Cohen 10291a86b377SEli Cohen /* Two QPs are used by each virtqueue. One is used by the driver and one by 10301a86b377SEli Cohen * firmware. The fw argument indicates whether the subjected QP is the one used 10311a86b377SEli Cohen * by firmware. 10321a86b377SEli Cohen */ 10331a86b377SEli Cohen static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) 10341a86b377SEli Cohen { 10351a86b377SEli Cohen int outlen; 10361a86b377SEli Cohen int inlen; 10371a86b377SEli Cohen void *out; 10381a86b377SEli Cohen void *in; 10391a86b377SEli Cohen int err; 10401a86b377SEli Cohen 10411a86b377SEli Cohen alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); 10421a86b377SEli Cohen if (!in || !out) 10431a86b377SEli Cohen return -ENOMEM; 10441a86b377SEli Cohen 10451a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); 10461a86b377SEli Cohen free_inout(in, out); 10471a86b377SEli Cohen return err; 10481a86b377SEli Cohen } 10491a86b377SEli Cohen 10501a86b377SEli Cohen static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 10511a86b377SEli Cohen { 10521a86b377SEli Cohen int err; 10531a86b377SEli Cohen 10541a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); 10551a86b377SEli Cohen if (err) 10561a86b377SEli Cohen return err; 10571a86b377SEli Cohen 10581a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); 10591a86b377SEli Cohen if (err) 10601a86b377SEli Cohen return err; 10611a86b377SEli Cohen 10621a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); 10631a86b377SEli Cohen if (err) 10641a86b377SEli Cohen return err; 10651a86b377SEli Cohen 10661a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); 10671a86b377SEli Cohen if (err) 10681a86b377SEli Cohen return err; 10691a86b377SEli Cohen 10701a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); 10711a86b377SEli Cohen if (err) 10721a86b377SEli Cohen return err; 10731a86b377SEli Cohen 10741a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); 10751a86b377SEli Cohen if (err) 10761a86b377SEli Cohen return err; 10771a86b377SEli Cohen 10781a86b377SEli Cohen return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); 10791a86b377SEli Cohen } 10801a86b377SEli Cohen 10811a86b377SEli Cohen struct mlx5_virtq_attr { 10821a86b377SEli Cohen u8 state; 10831a86b377SEli Cohen u16 available_index; 1084b35ccebeSEli Cohen u16 used_index; 10851a86b377SEli Cohen }; 10861a86b377SEli Cohen 10871a86b377SEli Cohen static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 10881a86b377SEli Cohen struct mlx5_virtq_attr *attr) 10891a86b377SEli Cohen { 10901a86b377SEli Cohen int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out); 10911a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {}; 10921a86b377SEli Cohen void *out; 10931a86b377SEli Cohen void *obj_context; 10941a86b377SEli Cohen void *cmd_hdr; 10951a86b377SEli Cohen int err; 10961a86b377SEli Cohen 10971a86b377SEli Cohen out = kzalloc(outlen, GFP_KERNEL); 10981a86b377SEli Cohen if (!out) 10991a86b377SEli Cohen return -ENOMEM; 11001a86b377SEli Cohen 11011a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr); 11021a86b377SEli Cohen 11031a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 11041a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 11051a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 11061a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 11071a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen); 11081a86b377SEli Cohen if (err) 11091a86b377SEli Cohen goto err_cmd; 11101a86b377SEli Cohen 11111a86b377SEli Cohen obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context); 11121a86b377SEli Cohen memset(attr, 0, sizeof(*attr)); 11131a86b377SEli Cohen attr->state = MLX5_GET(virtio_net_q_object, obj_context, state); 11141a86b377SEli Cohen attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index); 1115b35ccebeSEli Cohen attr->used_index = MLX5_GET(virtio_net_q_object, obj_context, hw_used_index); 11161a86b377SEli Cohen kfree(out); 11171a86b377SEli Cohen return 0; 11181a86b377SEli Cohen 11191a86b377SEli Cohen err_cmd: 11201a86b377SEli Cohen kfree(out); 11211a86b377SEli Cohen return err; 11221a86b377SEli Cohen } 11231a86b377SEli Cohen 11241a86b377SEli Cohen static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) 11251a86b377SEli Cohen { 11261a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); 11271a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {}; 11281a86b377SEli Cohen void *obj_context; 11291a86b377SEli Cohen void *cmd_hdr; 11301a86b377SEli Cohen void *in; 11311a86b377SEli Cohen int err; 11321a86b377SEli Cohen 11331a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 11341a86b377SEli Cohen if (!in) 11351a86b377SEli Cohen return -ENOMEM; 11361a86b377SEli Cohen 11371a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr); 11381a86b377SEli Cohen 11391a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); 11401a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 11411a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 11421a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 11431a86b377SEli Cohen 11441a86b377SEli Cohen obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context); 11451a86b377SEli Cohen MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select, 11461a86b377SEli Cohen MLX5_VIRTQ_MODIFY_MASK_STATE); 11471a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, state, state); 11481a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 11491a86b377SEli Cohen kfree(in); 11501a86b377SEli Cohen if (!err) 11511a86b377SEli Cohen mvq->fw_state = state; 11521a86b377SEli Cohen 11531a86b377SEli Cohen return err; 11541a86b377SEli Cohen } 11551a86b377SEli Cohen 11561892a3d4SEli Cohen static int counter_set_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11571892a3d4SEli Cohen { 11581892a3d4SEli Cohen u32 in[MLX5_ST_SZ_DW(create_virtio_q_counters_in)] = {}; 11591892a3d4SEli Cohen u32 out[MLX5_ST_SZ_DW(create_virtio_q_counters_out)] = {}; 11601892a3d4SEli Cohen void *cmd_hdr; 11611892a3d4SEli Cohen int err; 11621892a3d4SEli Cohen 11631892a3d4SEli Cohen if (!counters_supported(&ndev->mvdev)) 11641892a3d4SEli Cohen return 0; 11651892a3d4SEli Cohen 11661892a3d4SEli Cohen cmd_hdr = MLX5_ADDR_OF(create_virtio_q_counters_in, in, hdr); 11671892a3d4SEli Cohen 11681892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 11691892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 11701892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 11711892a3d4SEli Cohen 11721892a3d4SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 11731892a3d4SEli Cohen if (err) 11741892a3d4SEli Cohen return err; 11751892a3d4SEli Cohen 11761892a3d4SEli Cohen mvq->counter_set_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 11771892a3d4SEli Cohen 11781892a3d4SEli Cohen return 0; 11791892a3d4SEli Cohen } 11801892a3d4SEli Cohen 11811892a3d4SEli Cohen static void counter_set_dealloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11821892a3d4SEli Cohen { 11831892a3d4SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_virtio_q_counters_in)] = {}; 11841892a3d4SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_virtio_q_counters_out)] = {}; 11851892a3d4SEli Cohen 11861892a3d4SEli Cohen if (!counters_supported(&ndev->mvdev)) 11871892a3d4SEli Cohen return; 11881892a3d4SEli Cohen 11891892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 11901892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_id, mvq->counter_set_id); 11911892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.uid, ndev->mvdev.res.uid); 11921892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 11931892a3d4SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 11941892a3d4SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "dealloc counter set 0x%x\n", mvq->counter_set_id); 11951892a3d4SEli Cohen } 11961892a3d4SEli Cohen 11971a86b377SEli Cohen static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11981a86b377SEli Cohen { 11991a86b377SEli Cohen u16 idx = mvq->index; 12001a86b377SEli Cohen int err; 12011a86b377SEli Cohen 12021a86b377SEli Cohen if (!mvq->num_ent) 12031a86b377SEli Cohen return 0; 12041a86b377SEli Cohen 120552893733SEli Cohen if (mvq->initialized) 120652893733SEli Cohen return 0; 12071a86b377SEli Cohen 12081a86b377SEli Cohen err = cq_create(ndev, idx, mvq->num_ent); 12091a86b377SEli Cohen if (err) 12101a86b377SEli Cohen return err; 12111a86b377SEli Cohen 12121a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->fwqp); 12131a86b377SEli Cohen if (err) 12141a86b377SEli Cohen goto err_fwqp; 12151a86b377SEli Cohen 12161a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->vqqp); 12171a86b377SEli Cohen if (err) 12181a86b377SEli Cohen goto err_vqqp; 12191a86b377SEli Cohen 12201a86b377SEli Cohen err = connect_qps(ndev, mvq); 12211a86b377SEli Cohen if (err) 12221a86b377SEli Cohen goto err_connect; 12231a86b377SEli Cohen 12241892a3d4SEli Cohen err = counter_set_alloc(ndev, mvq); 12251892a3d4SEli Cohen if (err) 12261892a3d4SEli Cohen goto err_counter; 12271892a3d4SEli Cohen 12281a86b377SEli Cohen err = create_virtqueue(ndev, mvq); 12291a86b377SEli Cohen if (err) 12301a86b377SEli Cohen goto err_connect; 12311a86b377SEli Cohen 12321a86b377SEli Cohen if (mvq->ready) { 12331a86b377SEli Cohen err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 12341a86b377SEli Cohen if (err) { 12351a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", 12361a86b377SEli Cohen idx, err); 12371a86b377SEli Cohen goto err_connect; 12381a86b377SEli Cohen } 12391a86b377SEli Cohen } 12401a86b377SEli Cohen 12411a86b377SEli Cohen mvq->initialized = true; 12421a86b377SEli Cohen return 0; 12431a86b377SEli Cohen 12441a86b377SEli Cohen err_connect: 12451892a3d4SEli Cohen counter_set_dealloc(ndev, mvq); 12461892a3d4SEli Cohen err_counter: 12471a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 12481a86b377SEli Cohen err_vqqp: 12491a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 12501a86b377SEli Cohen err_fwqp: 12511a86b377SEli Cohen cq_destroy(ndev, idx); 12521a86b377SEli Cohen return err; 12531a86b377SEli Cohen } 12541a86b377SEli Cohen 12551a86b377SEli Cohen static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 12561a86b377SEli Cohen { 12571a86b377SEli Cohen struct mlx5_virtq_attr attr; 12581a86b377SEli Cohen 12591a86b377SEli Cohen if (!mvq->initialized) 12601a86b377SEli Cohen return; 12611a86b377SEli Cohen 12621a86b377SEli Cohen if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 12631a86b377SEli Cohen return; 12641a86b377SEli Cohen 12651a86b377SEli Cohen if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) 12661a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); 12673176e974SSi-Wei Liu 12683176e974SSi-Wei Liu if (query_virtqueue(ndev, mvq, &attr)) { 12693176e974SSi-Wei Liu mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); 12703176e974SSi-Wei Liu return; 12713176e974SSi-Wei Liu } 12723176e974SSi-Wei Liu mvq->avail_idx = attr.available_index; 1273bc04d93eSEli Cohen mvq->used_idx = attr.used_index; 12741a86b377SEli Cohen } 12751a86b377SEli Cohen 12761a86b377SEli Cohen static void suspend_vqs(struct mlx5_vdpa_net *ndev) 12771a86b377SEli Cohen { 12781a86b377SEli Cohen int i; 12791a86b377SEli Cohen 128075560522SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 12811a86b377SEli Cohen suspend_vq(ndev, &ndev->vqs[i]); 12821a86b377SEli Cohen } 12831a86b377SEli Cohen 12841a86b377SEli Cohen static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 12851a86b377SEli Cohen { 12861a86b377SEli Cohen if (!mvq->initialized) 12871a86b377SEli Cohen return; 12881a86b377SEli Cohen 12891a86b377SEli Cohen suspend_vq(ndev, mvq); 12901a86b377SEli Cohen destroy_virtqueue(ndev, mvq); 12911892a3d4SEli Cohen counter_set_dealloc(ndev, mvq); 12921a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 12931a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 12941a86b377SEli Cohen cq_destroy(ndev, mvq->index); 12951a86b377SEli Cohen mvq->initialized = false; 12961a86b377SEli Cohen } 12971a86b377SEli Cohen 12981a86b377SEli Cohen static int create_rqt(struct mlx5_vdpa_net *ndev) 12991a86b377SEli Cohen { 13001a86b377SEli Cohen __be32 *list; 13011a86b377SEli Cohen void *rqtc; 13021a86b377SEli Cohen int inlen; 13031a86b377SEli Cohen void *in; 13041a86b377SEli Cohen int i, j; 13051a86b377SEli Cohen int err; 13061a86b377SEli Cohen 1307acde3929SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + ndev->rqt_size * MLX5_ST_SZ_BYTES(rq_num); 13081a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 13091a86b377SEli Cohen if (!in) 13101a86b377SEli Cohen return -ENOMEM; 13111a86b377SEli Cohen 13121a86b377SEli Cohen MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); 13131a86b377SEli Cohen rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 13141a86b377SEli Cohen 13151a86b377SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 1316acde3929SEli Cohen MLX5_SET(rqtc, rqtc, rqt_max_size, ndev->rqt_size); 13171a86b377SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1318acde3929SEli Cohen for (i = 0, j = 0; i < ndev->rqt_size; i++, j += 2) 1319acde3929SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j % ndev->cur_num_vqs].virtq_id); 13201a86b377SEli Cohen 1321acde3929SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, ndev->rqt_size); 13221a86b377SEli Cohen err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); 13231a86b377SEli Cohen kfree(in); 13241a86b377SEli Cohen if (err) 13251a86b377SEli Cohen return err; 13261a86b377SEli Cohen 13271a86b377SEli Cohen return 0; 13281a86b377SEli Cohen } 13291a86b377SEli Cohen 133052893733SEli Cohen #define MLX5_MODIFY_RQT_NUM_RQS ((u64)1) 133152893733SEli Cohen 133252893733SEli Cohen static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) 133352893733SEli Cohen { 133452893733SEli Cohen __be32 *list; 133552893733SEli Cohen void *rqtc; 133652893733SEli Cohen int inlen; 133752893733SEli Cohen void *in; 133852893733SEli Cohen int i, j; 133952893733SEli Cohen int err; 134052893733SEli Cohen 1341acde3929SEli Cohen inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + ndev->rqt_size * MLX5_ST_SZ_BYTES(rq_num); 134252893733SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 134352893733SEli Cohen if (!in) 134452893733SEli Cohen return -ENOMEM; 134552893733SEli Cohen 134652893733SEli Cohen MLX5_SET(modify_rqt_in, in, uid, ndev->mvdev.res.uid); 134752893733SEli Cohen MLX5_SET64(modify_rqt_in, in, bitmask, MLX5_MODIFY_RQT_NUM_RQS); 134852893733SEli Cohen rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 134952893733SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 135052893733SEli Cohen 135152893733SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1352acde3929SEli Cohen for (i = 0, j = 0; i < ndev->rqt_size; i++, j += 2) 1353a7f46ba4SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j % num].virtq_id); 135452893733SEli Cohen 1355acde3929SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, ndev->rqt_size); 135652893733SEli Cohen err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn); 135752893733SEli Cohen kfree(in); 135852893733SEli Cohen if (err) 135952893733SEli Cohen return err; 136052893733SEli Cohen 136152893733SEli Cohen return 0; 136252893733SEli Cohen } 136352893733SEli Cohen 13641a86b377SEli Cohen static void destroy_rqt(struct mlx5_vdpa_net *ndev) 13651a86b377SEli Cohen { 13661a86b377SEli Cohen mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); 13671a86b377SEli Cohen } 13681a86b377SEli Cohen 13691a86b377SEli Cohen static int create_tir(struct mlx5_vdpa_net *ndev) 13701a86b377SEli Cohen { 13711a86b377SEli Cohen #define HASH_IP_L4PORTS \ 13721a86b377SEli Cohen (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \ 13731a86b377SEli Cohen MLX5_HASH_FIELD_SEL_L4_DPORT) 13741a86b377SEli Cohen static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 13751a86b377SEli Cohen 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 13761a86b377SEli Cohen 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 13771a86b377SEli Cohen 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 13781a86b377SEli Cohen 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a }; 13791a86b377SEli Cohen void *rss_key; 13801a86b377SEli Cohen void *outer; 13811a86b377SEli Cohen void *tirc; 13821a86b377SEli Cohen void *in; 13831a86b377SEli Cohen int err; 13841a86b377SEli Cohen 13851a86b377SEli Cohen in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL); 13861a86b377SEli Cohen if (!in) 13871a86b377SEli Cohen return -ENOMEM; 13881a86b377SEli Cohen 13891a86b377SEli Cohen MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); 13901a86b377SEli Cohen tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 13911a86b377SEli Cohen MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 13921a86b377SEli Cohen 13931a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 13941a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ); 13951a86b377SEli Cohen rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 13961a86b377SEli Cohen memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key)); 13971a86b377SEli Cohen 13981a86b377SEli Cohen outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 13991a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); 14001a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); 14011a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS); 14021a86b377SEli Cohen 14031a86b377SEli Cohen MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); 14041a86b377SEli Cohen MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); 14051a86b377SEli Cohen 14061a86b377SEli Cohen err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); 14071a86b377SEli Cohen kfree(in); 14081a86b377SEli Cohen return err; 14091a86b377SEli Cohen } 14101a86b377SEli Cohen 14111a86b377SEli Cohen static void destroy_tir(struct mlx5_vdpa_net *ndev) 14121a86b377SEli Cohen { 14131a86b377SEli Cohen mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); 14141a86b377SEli Cohen } 14151a86b377SEli Cohen 1416*baf2ad3fSEli Cohen #define MAX_STEERING_ENT 0x8000 1417*baf2ad3fSEli Cohen #define MAX_STEERING_GROUPS 2 1418*baf2ad3fSEli Cohen 1419*baf2ad3fSEli Cohen static int mlx5_vdpa_add_mac_vlan_rules(struct mlx5_vdpa_net *ndev, u8 *mac, 1420*baf2ad3fSEli Cohen u16 vid, bool tagged, 1421*baf2ad3fSEli Cohen struct mlx5_flow_handle **ucast, 1422*baf2ad3fSEli Cohen struct mlx5_flow_handle **mcast) 14231a86b377SEli Cohen { 14247becdd13SEli Cohen struct mlx5_flow_destination dest = {}; 14251a86b377SEli Cohen struct mlx5_flow_act flow_act = {}; 1426*baf2ad3fSEli Cohen struct mlx5_flow_handle *rule; 1427540061acSEli Cohen struct mlx5_flow_spec *spec; 1428540061acSEli Cohen void *headers_c; 1429540061acSEli Cohen void *headers_v; 1430540061acSEli Cohen u8 *dmac_c; 1431540061acSEli Cohen u8 *dmac_v; 14321a86b377SEli Cohen int err; 14331a86b377SEli Cohen 1434540061acSEli Cohen spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1435540061acSEli Cohen if (!spec) 1436540061acSEli Cohen return -ENOMEM; 1437540061acSEli Cohen 1438540061acSEli Cohen spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1439540061acSEli Cohen headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers); 1440540061acSEli Cohen headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); 1441*baf2ad3fSEli Cohen dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, outer_headers.dmac_47_16); 1442540061acSEli Cohen dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16); 1443*baf2ad3fSEli Cohen memset(dmac_c, 0xff, ETH_ALEN); 1444*baf2ad3fSEli Cohen ether_addr_copy(dmac_v, mac); 1445*baf2ad3fSEli Cohen MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1); 1446*baf2ad3fSEli Cohen if (tagged) { 1447*baf2ad3fSEli Cohen MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); 1448*baf2ad3fSEli Cohen MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, first_vid); 1449*baf2ad3fSEli Cohen MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_vid, vid); 1450*baf2ad3fSEli Cohen } 14517becdd13SEli Cohen flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 14527becdd13SEli Cohen dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 14537becdd13SEli Cohen dest.tir_num = ndev->res.tirn; 1454*baf2ad3fSEli Cohen rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, &dest, 1); 1455*baf2ad3fSEli Cohen if (IS_ERR(rule)) 1456*baf2ad3fSEli Cohen return PTR_ERR(rule); 1457540061acSEli Cohen 1458*baf2ad3fSEli Cohen *ucast = rule; 14591a86b377SEli Cohen 1460540061acSEli Cohen memset(dmac_c, 0, ETH_ALEN); 1461540061acSEli Cohen memset(dmac_v, 0, ETH_ALEN); 1462540061acSEli Cohen dmac_c[0] = 1; 1463540061acSEli Cohen dmac_v[0] = 1; 1464*baf2ad3fSEli Cohen rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, &dest, 1); 1465*baf2ad3fSEli Cohen kvfree(spec); 1466*baf2ad3fSEli Cohen if (IS_ERR(rule)) { 1467*baf2ad3fSEli Cohen err = PTR_ERR(rule); 1468*baf2ad3fSEli Cohen goto err_mcast; 1469540061acSEli Cohen } 1470540061acSEli Cohen 1471*baf2ad3fSEli Cohen *mcast = rule; 14721a86b377SEli Cohen return 0; 14731a86b377SEli Cohen 1474*baf2ad3fSEli Cohen err_mcast: 1475*baf2ad3fSEli Cohen mlx5_del_flow_rules(*ucast); 14761a86b377SEli Cohen return err; 14771a86b377SEli Cohen } 14781a86b377SEli Cohen 1479*baf2ad3fSEli Cohen static void mlx5_vdpa_del_mac_vlan_rules(struct mlx5_vdpa_net *ndev, 1480*baf2ad3fSEli Cohen struct mlx5_flow_handle *ucast, 1481*baf2ad3fSEli Cohen struct mlx5_flow_handle *mcast) 14821a86b377SEli Cohen { 1483*baf2ad3fSEli Cohen mlx5_del_flow_rules(ucast); 1484*baf2ad3fSEli Cohen mlx5_del_flow_rules(mcast); 1485*baf2ad3fSEli Cohen } 1486*baf2ad3fSEli Cohen 1487*baf2ad3fSEli Cohen static u64 search_val(u8 *mac, u16 vlan, bool tagged) 1488*baf2ad3fSEli Cohen { 1489*baf2ad3fSEli Cohen u64 val; 1490*baf2ad3fSEli Cohen 1491*baf2ad3fSEli Cohen if (!tagged) 1492*baf2ad3fSEli Cohen vlan = MLX5V_UNTAGGED; 1493*baf2ad3fSEli Cohen 1494*baf2ad3fSEli Cohen val = (u64)vlan << 48 | 1495*baf2ad3fSEli Cohen (u64)mac[0] << 40 | 1496*baf2ad3fSEli Cohen (u64)mac[1] << 32 | 1497*baf2ad3fSEli Cohen (u64)mac[2] << 24 | 1498*baf2ad3fSEli Cohen (u64)mac[3] << 16 | 1499*baf2ad3fSEli Cohen (u64)mac[4] << 8 | 1500*baf2ad3fSEli Cohen (u64)mac[5]; 1501*baf2ad3fSEli Cohen 1502*baf2ad3fSEli Cohen return val; 1503*baf2ad3fSEli Cohen } 1504*baf2ad3fSEli Cohen 1505*baf2ad3fSEli Cohen static struct macvlan_node *mac_vlan_lookup(struct mlx5_vdpa_net *ndev, u64 value) 1506*baf2ad3fSEli Cohen { 1507*baf2ad3fSEli Cohen struct macvlan_node *pos; 1508*baf2ad3fSEli Cohen u32 idx; 1509*baf2ad3fSEli Cohen 1510*baf2ad3fSEli Cohen idx = hash_64(value, 8); // tbd 8 1511*baf2ad3fSEli Cohen hlist_for_each_entry(pos, &ndev->macvlan_hash[idx], hlist) { 1512*baf2ad3fSEli Cohen if (pos->macvlan == value) 1513*baf2ad3fSEli Cohen return pos; 1514*baf2ad3fSEli Cohen } 1515*baf2ad3fSEli Cohen return NULL; 1516*baf2ad3fSEli Cohen } 1517*baf2ad3fSEli Cohen 1518*baf2ad3fSEli Cohen static int mac_vlan_add(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vlan, bool tagged) // vlan -> vid 1519*baf2ad3fSEli Cohen { 1520*baf2ad3fSEli Cohen struct macvlan_node *ptr; 1521*baf2ad3fSEli Cohen u64 val; 1522*baf2ad3fSEli Cohen u32 idx; 1523*baf2ad3fSEli Cohen int err; 1524*baf2ad3fSEli Cohen 1525*baf2ad3fSEli Cohen val = search_val(mac, vlan, tagged); 1526*baf2ad3fSEli Cohen if (mac_vlan_lookup(ndev, val)) 1527*baf2ad3fSEli Cohen return -EEXIST; 1528*baf2ad3fSEli Cohen 1529*baf2ad3fSEli Cohen ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); 1530*baf2ad3fSEli Cohen if (!ptr) 1531*baf2ad3fSEli Cohen return -ENOMEM; 1532*baf2ad3fSEli Cohen 1533*baf2ad3fSEli Cohen err = mlx5_vdpa_add_mac_vlan_rules(ndev, ndev->config.mac, vlan, tagged, 1534*baf2ad3fSEli Cohen &ptr->ucast_rule, &ptr->mcast_rule); 1535*baf2ad3fSEli Cohen if (err) 1536*baf2ad3fSEli Cohen goto err_add; 1537*baf2ad3fSEli Cohen 1538*baf2ad3fSEli Cohen ptr->macvlan = val; 1539*baf2ad3fSEli Cohen idx = hash_64(val, 8); 1540*baf2ad3fSEli Cohen hlist_add_head(&ptr->hlist, &ndev->macvlan_hash[idx]); 1541*baf2ad3fSEli Cohen return 0; 1542*baf2ad3fSEli Cohen 1543*baf2ad3fSEli Cohen err_add: 1544*baf2ad3fSEli Cohen kfree(ptr); 1545*baf2ad3fSEli Cohen return err; 1546*baf2ad3fSEli Cohen } 1547*baf2ad3fSEli Cohen 1548*baf2ad3fSEli Cohen static void mac_vlan_del(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vlan, bool tagged) 1549*baf2ad3fSEli Cohen { 1550*baf2ad3fSEli Cohen struct macvlan_node *ptr; 1551*baf2ad3fSEli Cohen 1552*baf2ad3fSEli Cohen ptr = mac_vlan_lookup(ndev, search_val(mac, vlan, tagged)); 1553*baf2ad3fSEli Cohen if (!ptr) 15541a86b377SEli Cohen return; 15551a86b377SEli Cohen 1556*baf2ad3fSEli Cohen hlist_del(&ptr->hlist); 1557*baf2ad3fSEli Cohen mlx5_vdpa_del_mac_vlan_rules(ndev, ptr->ucast_rule, ptr->mcast_rule); 1558*baf2ad3fSEli Cohen kfree(ptr); 1559*baf2ad3fSEli Cohen } 1560*baf2ad3fSEli Cohen 1561*baf2ad3fSEli Cohen static void clear_mac_vlan_table(struct mlx5_vdpa_net *ndev) 1562*baf2ad3fSEli Cohen { 1563*baf2ad3fSEli Cohen struct macvlan_node *pos; 1564*baf2ad3fSEli Cohen struct hlist_node *n; 1565*baf2ad3fSEli Cohen int i; 1566*baf2ad3fSEli Cohen 1567*baf2ad3fSEli Cohen for (i = 0; i < MLX5V_MACVLAN_SIZE; i++) { 1568*baf2ad3fSEli Cohen hlist_for_each_entry_safe(pos, n, &ndev->macvlan_hash[i], hlist) { 1569*baf2ad3fSEli Cohen hlist_del(&pos->hlist); 1570*baf2ad3fSEli Cohen mlx5_vdpa_del_mac_vlan_rules(ndev, pos->ucast_rule, pos->mcast_rule); 1571*baf2ad3fSEli Cohen kfree(pos); 1572*baf2ad3fSEli Cohen } 1573*baf2ad3fSEli Cohen } 1574*baf2ad3fSEli Cohen } 1575*baf2ad3fSEli Cohen 1576*baf2ad3fSEli Cohen static int setup_steering(struct mlx5_vdpa_net *ndev) 1577*baf2ad3fSEli Cohen { 1578*baf2ad3fSEli Cohen struct mlx5_flow_table_attr ft_attr = {}; 1579*baf2ad3fSEli Cohen struct mlx5_flow_namespace *ns; 1580*baf2ad3fSEli Cohen int err; 1581*baf2ad3fSEli Cohen 1582*baf2ad3fSEli Cohen ft_attr.max_fte = MAX_STEERING_ENT; 1583*baf2ad3fSEli Cohen ft_attr.autogroup.max_num_groups = MAX_STEERING_GROUPS; 1584*baf2ad3fSEli Cohen 1585*baf2ad3fSEli Cohen ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); 1586*baf2ad3fSEli Cohen if (!ns) { 1587*baf2ad3fSEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to get flow namespace\n"); 1588*baf2ad3fSEli Cohen return -EOPNOTSUPP; 1589*baf2ad3fSEli Cohen } 1590*baf2ad3fSEli Cohen 1591*baf2ad3fSEli Cohen ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 1592*baf2ad3fSEli Cohen if (IS_ERR(ndev->rxft)) { 1593*baf2ad3fSEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to create flow table\n"); 1594*baf2ad3fSEli Cohen return PTR_ERR(ndev->rxft); 1595*baf2ad3fSEli Cohen } 1596*baf2ad3fSEli Cohen 1597*baf2ad3fSEli Cohen err = mac_vlan_add(ndev, ndev->config.mac, 0, false); 1598*baf2ad3fSEli Cohen if (err) 1599*baf2ad3fSEli Cohen goto err_add; 1600*baf2ad3fSEli Cohen 1601*baf2ad3fSEli Cohen return 0; 1602*baf2ad3fSEli Cohen 1603*baf2ad3fSEli Cohen err_add: 1604*baf2ad3fSEli Cohen mlx5_destroy_flow_table(ndev->rxft); 1605*baf2ad3fSEli Cohen return err; 1606*baf2ad3fSEli Cohen } 1607*baf2ad3fSEli Cohen 1608*baf2ad3fSEli Cohen static void teardown_steering(struct mlx5_vdpa_net *ndev) 1609*baf2ad3fSEli Cohen { 1610*baf2ad3fSEli Cohen clear_mac_vlan_table(ndev); 16111a86b377SEli Cohen mlx5_destroy_flow_table(ndev->rxft); 16121a86b377SEli Cohen } 16131a86b377SEli Cohen 16145262912eSEli Cohen static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) 16155262912eSEli Cohen { 16165262912eSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 16175262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 16185262912eSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 16195262912eSEli Cohen struct mlx5_core_dev *pfmdev; 16205262912eSEli Cohen size_t read; 1621f1781bedSMichael Qiu u8 mac[ETH_ALEN], mac_back[ETH_ALEN]; 16225262912eSEli Cohen 16235262912eSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 16245262912eSEli Cohen switch (cmd) { 16255262912eSEli Cohen case VIRTIO_NET_CTRL_MAC_ADDR_SET: 16265262912eSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)mac, ETH_ALEN); 16275262912eSEli Cohen if (read != ETH_ALEN) 16285262912eSEli Cohen break; 16295262912eSEli Cohen 16305262912eSEli Cohen if (!memcmp(ndev->config.mac, mac, 6)) { 16315262912eSEli Cohen status = VIRTIO_NET_OK; 16325262912eSEli Cohen break; 16335262912eSEli Cohen } 16345262912eSEli Cohen 1635f1781bedSMichael Qiu if (is_zero_ether_addr(mac)) 1636f1781bedSMichael Qiu break; 1637f1781bedSMichael Qiu 16385262912eSEli Cohen if (!is_zero_ether_addr(ndev->config.mac)) { 16395262912eSEli Cohen if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 16405262912eSEli Cohen mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n", 16415262912eSEli Cohen ndev->config.mac); 16425262912eSEli Cohen break; 16435262912eSEli Cohen } 16445262912eSEli Cohen } 16455262912eSEli Cohen 16465262912eSEli Cohen if (mlx5_mpfs_add_mac(pfmdev, mac)) { 16475262912eSEli Cohen mlx5_vdpa_warn(mvdev, "failed to insert new MAC %pM into MPFS table\n", 16485262912eSEli Cohen mac); 16495262912eSEli Cohen break; 16505262912eSEli Cohen } 16515262912eSEli Cohen 1652f1781bedSMichael Qiu /* backup the original mac address so that if failed to add the forward rules 1653f1781bedSMichael Qiu * we could restore it 1654f1781bedSMichael Qiu */ 1655f1781bedSMichael Qiu memcpy(mac_back, ndev->config.mac, ETH_ALEN); 1656f1781bedSMichael Qiu 16575262912eSEli Cohen memcpy(ndev->config.mac, mac, ETH_ALEN); 1658f1781bedSMichael Qiu 1659f1781bedSMichael Qiu /* Need recreate the flow table entry, so that the packet could forward back 1660f1781bedSMichael Qiu */ 1661*baf2ad3fSEli Cohen mac_vlan_del(ndev, ndev->config.mac, 0, false); 1662f1781bedSMichael Qiu 1663*baf2ad3fSEli Cohen if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) { 1664f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "failed to insert forward rules, try to restore\n"); 1665f1781bedSMichael Qiu 1666f1781bedSMichael Qiu /* Although it hardly run here, we still need double check */ 1667f1781bedSMichael Qiu if (is_zero_ether_addr(mac_back)) { 1668f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore mac failed: Original MAC is zero\n"); 1669f1781bedSMichael Qiu break; 1670f1781bedSMichael Qiu } 1671f1781bedSMichael Qiu 1672f1781bedSMichael Qiu /* Try to restore original mac address to MFPS table, and try to restore 1673f1781bedSMichael Qiu * the forward rule entry. 1674f1781bedSMichael Qiu */ 1675f1781bedSMichael Qiu if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 1676f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore mac failed: delete MAC %pM from MPFS table failed\n", 1677f1781bedSMichael Qiu ndev->config.mac); 1678f1781bedSMichael Qiu } 1679f1781bedSMichael Qiu 1680f1781bedSMichael Qiu if (mlx5_mpfs_add_mac(pfmdev, mac_back)) { 1681f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore mac failed: insert old MAC %pM into MPFS table failed\n", 1682f1781bedSMichael Qiu mac_back); 1683f1781bedSMichael Qiu } 1684f1781bedSMichael Qiu 1685f1781bedSMichael Qiu memcpy(ndev->config.mac, mac_back, ETH_ALEN); 1686f1781bedSMichael Qiu 1687*baf2ad3fSEli Cohen if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) 1688f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore forward rules failed: insert forward rules failed\n"); 1689f1781bedSMichael Qiu 1690f1781bedSMichael Qiu break; 1691f1781bedSMichael Qiu } 1692f1781bedSMichael Qiu 16935262912eSEli Cohen status = VIRTIO_NET_OK; 16945262912eSEli Cohen break; 16955262912eSEli Cohen 16965262912eSEli Cohen default: 16975262912eSEli Cohen break; 16985262912eSEli Cohen } 16995262912eSEli Cohen 17005262912eSEli Cohen return status; 17015262912eSEli Cohen } 17025262912eSEli Cohen 170352893733SEli Cohen static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps) 170452893733SEli Cohen { 170552893733SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 170652893733SEli Cohen int cur_qps = ndev->cur_num_vqs / 2; 170752893733SEli Cohen int err; 170852893733SEli Cohen int i; 170952893733SEli Cohen 171052893733SEli Cohen if (cur_qps > newqps) { 171152893733SEli Cohen err = modify_rqt(ndev, 2 * newqps); 171252893733SEli Cohen if (err) 171352893733SEli Cohen return err; 171452893733SEli Cohen 171552893733SEli Cohen for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) 171652893733SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 171752893733SEli Cohen 171852893733SEli Cohen ndev->cur_num_vqs = 2 * newqps; 171952893733SEli Cohen } else { 172052893733SEli Cohen ndev->cur_num_vqs = 2 * newqps; 172152893733SEli Cohen for (i = cur_qps * 2; i < 2 * newqps; i++) { 172252893733SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 172352893733SEli Cohen if (err) 172452893733SEli Cohen goto clean_added; 172552893733SEli Cohen } 172652893733SEli Cohen err = modify_rqt(ndev, 2 * newqps); 172752893733SEli Cohen if (err) 172852893733SEli Cohen goto clean_added; 172952893733SEli Cohen } 173052893733SEli Cohen return 0; 173152893733SEli Cohen 173252893733SEli Cohen clean_added: 173337e07e70SEli Cohen for (--i; i >= 2 * cur_qps; --i) 173452893733SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 173552893733SEli Cohen 173637e07e70SEli Cohen ndev->cur_num_vqs = 2 * cur_qps; 173737e07e70SEli Cohen 173852893733SEli Cohen return err; 173952893733SEli Cohen } 174052893733SEli Cohen 174152893733SEli Cohen static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd) 174252893733SEli Cohen { 174352893733SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 174452893733SEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 174552893733SEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 174652893733SEli Cohen struct virtio_net_ctrl_mq mq; 174752893733SEli Cohen size_t read; 174852893733SEli Cohen u16 newqps; 174952893733SEli Cohen 175052893733SEli Cohen switch (cmd) { 175152893733SEli Cohen case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: 1752ed0f849fSSi-Wei Liu /* This mq feature check aligns with pre-existing userspace 1753ed0f849fSSi-Wei Liu * implementation. 1754ed0f849fSSi-Wei Liu * 1755ed0f849fSSi-Wei Liu * Without it, an untrusted driver could fake a multiqueue config 1756ed0f849fSSi-Wei Liu * request down to a non-mq device that may cause kernel to 1757ed0f849fSSi-Wei Liu * panic due to uninitialized resources for extra vqs. Even with 1758ed0f849fSSi-Wei Liu * a well behaving guest driver, it is not expected to allow 1759ed0f849fSSi-Wei Liu * changing the number of vqs on a non-mq device. 1760ed0f849fSSi-Wei Liu */ 1761ed0f849fSSi-Wei Liu if (!MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) 1762ed0f849fSSi-Wei Liu break; 1763ed0f849fSSi-Wei Liu 176452893733SEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)&mq, sizeof(mq)); 176552893733SEli Cohen if (read != sizeof(mq)) 176652893733SEli Cohen break; 176752893733SEli Cohen 176852893733SEli Cohen newqps = mlx5vdpa16_to_cpu(mvdev, mq.virtqueue_pairs); 1769ed0f849fSSi-Wei Liu if (newqps < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 1770acde3929SEli Cohen newqps > ndev->rqt_size) 1771ed0f849fSSi-Wei Liu break; 1772ed0f849fSSi-Wei Liu 177352893733SEli Cohen if (ndev->cur_num_vqs == 2 * newqps) { 177452893733SEli Cohen status = VIRTIO_NET_OK; 177552893733SEli Cohen break; 177652893733SEli Cohen } 177752893733SEli Cohen 177852893733SEli Cohen if (!change_num_qps(mvdev, newqps)) 177952893733SEli Cohen status = VIRTIO_NET_OK; 178052893733SEli Cohen 178152893733SEli Cohen break; 178252893733SEli Cohen default: 178352893733SEli Cohen break; 178452893733SEli Cohen } 178552893733SEli Cohen 178652893733SEli Cohen return status; 178752893733SEli Cohen } 178852893733SEli Cohen 1789*baf2ad3fSEli Cohen static virtio_net_ctrl_ack handle_ctrl_vlan(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1790*baf2ad3fSEli Cohen { 1791*baf2ad3fSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1792*baf2ad3fSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1793*baf2ad3fSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 1794*baf2ad3fSEli Cohen __virtio16 vlan; 1795*baf2ad3fSEli Cohen size_t read; 1796*baf2ad3fSEli Cohen u16 id; 1797*baf2ad3fSEli Cohen 1798*baf2ad3fSEli Cohen switch (cmd) { 1799*baf2ad3fSEli Cohen case VIRTIO_NET_CTRL_VLAN_ADD: 1800*baf2ad3fSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &vlan, sizeof(vlan)); 1801*baf2ad3fSEli Cohen if (read != sizeof(vlan)) 1802*baf2ad3fSEli Cohen break; 1803*baf2ad3fSEli Cohen 1804*baf2ad3fSEli Cohen id = mlx5vdpa16_to_cpu(mvdev, vlan); 1805*baf2ad3fSEli Cohen if (mac_vlan_add(ndev, ndev->config.mac, id, true)) 1806*baf2ad3fSEli Cohen break; 1807*baf2ad3fSEli Cohen 1808*baf2ad3fSEli Cohen status = VIRTIO_NET_OK; 1809*baf2ad3fSEli Cohen break; 1810*baf2ad3fSEli Cohen case VIRTIO_NET_CTRL_VLAN_DEL: 1811*baf2ad3fSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &vlan, sizeof(vlan)); 1812*baf2ad3fSEli Cohen if (read != sizeof(vlan)) 1813*baf2ad3fSEli Cohen break; 1814*baf2ad3fSEli Cohen 1815*baf2ad3fSEli Cohen id = mlx5vdpa16_to_cpu(mvdev, vlan); 1816*baf2ad3fSEli Cohen mac_vlan_del(ndev, ndev->config.mac, id, true); 1817*baf2ad3fSEli Cohen break; 1818*baf2ad3fSEli Cohen default: 1819*baf2ad3fSEli Cohen break; 1820*baf2ad3fSEli Cohen } 1821*baf2ad3fSEli Cohen 1822*baf2ad3fSEli Cohen return status; 1823*baf2ad3fSEli Cohen } 1824*baf2ad3fSEli Cohen 18255262912eSEli Cohen static void mlx5_cvq_kick_handler(struct work_struct *work) 18265262912eSEli Cohen { 18275262912eSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 18285262912eSEli Cohen struct virtio_net_ctrl_hdr ctrl; 1829218bdd20SEli Cohen struct mlx5_vdpa_wq_ent *wqent; 18305262912eSEli Cohen struct mlx5_vdpa_dev *mvdev; 18315262912eSEli Cohen struct mlx5_control_vq *cvq; 18325262912eSEli Cohen struct mlx5_vdpa_net *ndev; 18335262912eSEli Cohen size_t read, write; 18345262912eSEli Cohen int err; 18355262912eSEli Cohen 1836218bdd20SEli Cohen wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 18375262912eSEli Cohen mvdev = wqent->mvdev; 18385262912eSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 18395262912eSEli Cohen cvq = &mvdev->cvq; 18401c80cf03SJason Wang 1841759ae7f9SEli Cohen down_write(&ndev->reslock); 18421c80cf03SJason Wang 18431c80cf03SJason Wang if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 18441c80cf03SJason Wang goto out; 18451c80cf03SJason Wang 18465262912eSEli Cohen if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 18471c80cf03SJason Wang goto out; 18485262912eSEli Cohen 18495262912eSEli Cohen if (!cvq->ready) 18501c80cf03SJason Wang goto out; 18515262912eSEli Cohen 18525262912eSEli Cohen while (true) { 18535262912eSEli Cohen err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head, 18545262912eSEli Cohen GFP_ATOMIC); 18555262912eSEli Cohen if (err <= 0) 18565262912eSEli Cohen break; 18575262912eSEli Cohen 18585262912eSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &ctrl, sizeof(ctrl)); 18595262912eSEli Cohen if (read != sizeof(ctrl)) 18605262912eSEli Cohen break; 18615262912eSEli Cohen 18621892a3d4SEli Cohen cvq->received_desc++; 18635262912eSEli Cohen switch (ctrl.class) { 18645262912eSEli Cohen case VIRTIO_NET_CTRL_MAC: 18655262912eSEli Cohen status = handle_ctrl_mac(mvdev, ctrl.cmd); 18665262912eSEli Cohen break; 186752893733SEli Cohen case VIRTIO_NET_CTRL_MQ: 186852893733SEli Cohen status = handle_ctrl_mq(mvdev, ctrl.cmd); 186952893733SEli Cohen break; 1870*baf2ad3fSEli Cohen case VIRTIO_NET_CTRL_VLAN: 1871*baf2ad3fSEli Cohen status = handle_ctrl_vlan(mvdev, ctrl.cmd); 1872*baf2ad3fSEli Cohen break; 18735262912eSEli Cohen default: 18745262912eSEli Cohen break; 18755262912eSEli Cohen } 18765262912eSEli Cohen 18775262912eSEli Cohen /* Make sure data is written before advancing index */ 18785262912eSEli Cohen smp_wmb(); 18795262912eSEli Cohen 18805262912eSEli Cohen write = vringh_iov_push_iotlb(&cvq->vring, &cvq->wiov, &status, sizeof(status)); 18815262912eSEli Cohen vringh_complete_iotlb(&cvq->vring, cvq->head, write); 18825262912eSEli Cohen vringh_kiov_cleanup(&cvq->riov); 18835262912eSEli Cohen vringh_kiov_cleanup(&cvq->wiov); 18845262912eSEli Cohen 18855262912eSEli Cohen if (vringh_need_notify_iotlb(&cvq->vring)) 18865262912eSEli Cohen vringh_notify(&cvq->vring); 188755ebf0d6SJason Wang 18881892a3d4SEli Cohen cvq->completed_desc++; 188955ebf0d6SJason Wang queue_work(mvdev->wq, &wqent->work); 189055ebf0d6SJason Wang break; 18915262912eSEli Cohen } 18921c80cf03SJason Wang 18931c80cf03SJason Wang out: 1894759ae7f9SEli Cohen up_write(&ndev->reslock); 18955262912eSEli Cohen } 18965262912eSEli Cohen 18971a86b377SEli Cohen static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) 18981a86b377SEli Cohen { 18991a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19001a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19015262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 19021a86b377SEli Cohen 1903e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1904e4fc6650SEli Cohen return; 1905e4fc6650SEli Cohen 19065262912eSEli Cohen if (unlikely(is_ctrl_vq_idx(mvdev, idx))) { 1907ad6dc1daSEli Cohen if (!mvdev->wq || !mvdev->cvq.ready) 19085262912eSEli Cohen return; 19095262912eSEli Cohen 191055ebf0d6SJason Wang queue_work(mvdev->wq, &ndev->cvq_ent.work); 19115262912eSEli Cohen return; 19125262912eSEli Cohen } 19135262912eSEli Cohen 19145262912eSEli Cohen mvq = &ndev->vqs[idx]; 19151a86b377SEli Cohen if (unlikely(!mvq->ready)) 19161a86b377SEli Cohen return; 19171a86b377SEli Cohen 19181a86b377SEli Cohen iowrite16(idx, ndev->mvdev.res.kick_addr); 19191a86b377SEli Cohen } 19201a86b377SEli Cohen 19211a86b377SEli Cohen static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 19221a86b377SEli Cohen u64 driver_area, u64 device_area) 19231a86b377SEli Cohen { 19241a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19251a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19265262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 19271a86b377SEli Cohen 1928e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1929e4fc6650SEli Cohen return -EINVAL; 1930e4fc6650SEli Cohen 19315262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 19325262912eSEli Cohen mvdev->cvq.desc_addr = desc_area; 19335262912eSEli Cohen mvdev->cvq.device_addr = device_area; 19345262912eSEli Cohen mvdev->cvq.driver_addr = driver_area; 19355262912eSEli Cohen return 0; 19365262912eSEli Cohen } 19375262912eSEli Cohen 19385262912eSEli Cohen mvq = &ndev->vqs[idx]; 19391a86b377SEli Cohen mvq->desc_addr = desc_area; 19401a86b377SEli Cohen mvq->device_addr = device_area; 19411a86b377SEli Cohen mvq->driver_addr = driver_area; 19421a86b377SEli Cohen return 0; 19431a86b377SEli Cohen } 19441a86b377SEli Cohen 19451a86b377SEli Cohen static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 19461a86b377SEli Cohen { 19471a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19481a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19491a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 19501a86b377SEli Cohen 19515262912eSEli Cohen if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 1952e4fc6650SEli Cohen return; 1953e4fc6650SEli Cohen 19541a86b377SEli Cohen mvq = &ndev->vqs[idx]; 19551a86b377SEli Cohen mvq->num_ent = num; 19561a86b377SEli Cohen } 19571a86b377SEli Cohen 19581a86b377SEli Cohen static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 19591a86b377SEli Cohen { 19601a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19611a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19621a86b377SEli Cohen 1963db296d25SEli Cohen ndev->event_cbs[idx] = *cb; 19641a86b377SEli Cohen } 19651a86b377SEli Cohen 19665262912eSEli Cohen static void mlx5_cvq_notify(struct vringh *vring) 19675262912eSEli Cohen { 19685262912eSEli Cohen struct mlx5_control_vq *cvq = container_of(vring, struct mlx5_control_vq, vring); 19695262912eSEli Cohen 19705262912eSEli Cohen if (!cvq->event_cb.callback) 19715262912eSEli Cohen return; 19725262912eSEli Cohen 19735262912eSEli Cohen cvq->event_cb.callback(cvq->event_cb.private); 19745262912eSEli Cohen } 19755262912eSEli Cohen 19765262912eSEli Cohen static void set_cvq_ready(struct mlx5_vdpa_dev *mvdev, bool ready) 19775262912eSEli Cohen { 19785262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 19795262912eSEli Cohen 19805262912eSEli Cohen cvq->ready = ready; 19815262912eSEli Cohen if (!ready) 19825262912eSEli Cohen return; 19835262912eSEli Cohen 19845262912eSEli Cohen cvq->vring.notify = mlx5_cvq_notify; 19851a86b377SEli Cohen } 19861a86b377SEli Cohen 19871a86b377SEli Cohen static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 19881a86b377SEli Cohen { 19891a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19901a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19915262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 19921a86b377SEli Cohen 1993759be899SEli Cohen if (!mvdev->actual_features) 1994759be899SEli Cohen return; 1995759be899SEli Cohen 1996e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1997e4fc6650SEli Cohen return; 1998e4fc6650SEli Cohen 19995262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 20005262912eSEli Cohen set_cvq_ready(mvdev, ready); 20015262912eSEli Cohen return; 20025262912eSEli Cohen } 20035262912eSEli Cohen 20045262912eSEli Cohen mvq = &ndev->vqs[idx]; 20051a86b377SEli Cohen if (!ready) 20061a86b377SEli Cohen suspend_vq(ndev, mvq); 20071a86b377SEli Cohen 20081a86b377SEli Cohen mvq->ready = ready; 20091a86b377SEli Cohen } 20101a86b377SEli Cohen 20111a86b377SEli Cohen static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx) 20121a86b377SEli Cohen { 20131a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 20141a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20151a86b377SEli Cohen 2016e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 2017e4fc6650SEli Cohen return false; 2018e4fc6650SEli Cohen 20195262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) 20205262912eSEli Cohen return mvdev->cvq.ready; 20215262912eSEli Cohen 20225262912eSEli Cohen return ndev->vqs[idx].ready; 20231a86b377SEli Cohen } 20241a86b377SEli Cohen 20251a86b377SEli Cohen static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx, 20261a86b377SEli Cohen const struct vdpa_vq_state *state) 20271a86b377SEli Cohen { 20281a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 20291a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20305262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 20311a86b377SEli Cohen 2032e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 2033e4fc6650SEli Cohen return -EINVAL; 2034e4fc6650SEli Cohen 20355262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 20365262912eSEli Cohen mvdev->cvq.vring.last_avail_idx = state->split.avail_index; 20375262912eSEli Cohen return 0; 20385262912eSEli Cohen } 20395262912eSEli Cohen 20405262912eSEli Cohen mvq = &ndev->vqs[idx]; 20411a86b377SEli Cohen if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) { 20421a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "can't modify available index\n"); 20431a86b377SEli Cohen return -EINVAL; 20441a86b377SEli Cohen } 20451a86b377SEli Cohen 2046530a5678SJason Wang mvq->used_idx = state->split.avail_index; 2047530a5678SJason Wang mvq->avail_idx = state->split.avail_index; 20481a86b377SEli Cohen return 0; 20491a86b377SEli Cohen } 20501a86b377SEli Cohen 20511a86b377SEli Cohen static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 20521a86b377SEli Cohen { 20531a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 20541a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20555262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 20561a86b377SEli Cohen struct mlx5_virtq_attr attr; 20571a86b377SEli Cohen int err; 20581a86b377SEli Cohen 2059e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 2060e4fc6650SEli Cohen return -EINVAL; 2061e4fc6650SEli Cohen 20625262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 20635262912eSEli Cohen state->split.avail_index = mvdev->cvq.vring.last_avail_idx; 20645262912eSEli Cohen return 0; 20655262912eSEli Cohen } 20665262912eSEli Cohen 20675262912eSEli Cohen mvq = &ndev->vqs[idx]; 20683176e974SSi-Wei Liu /* If the virtq object was destroyed, use the value saved at 20693176e974SSi-Wei Liu * the last minute of suspend_vq. This caters for userspace 20703176e974SSi-Wei Liu * that cares about emulating the index after vq is stopped. 20713176e974SSi-Wei Liu */ 20723176e974SSi-Wei Liu if (!mvq->initialized) { 2073bc04d93eSEli Cohen /* Firmware returns a wrong value for the available index. 2074bc04d93eSEli Cohen * Since both values should be identical, we take the value of 2075bc04d93eSEli Cohen * used_idx which is reported correctly. 2076bc04d93eSEli Cohen */ 2077530a5678SJason Wang state->split.avail_index = mvq->used_idx; 20783176e974SSi-Wei Liu return 0; 20793176e974SSi-Wei Liu } 20801a86b377SEli Cohen 20811a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 20821a86b377SEli Cohen if (err) { 20831a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n"); 20841a86b377SEli Cohen return err; 20851a86b377SEli Cohen } 2086530a5678SJason Wang state->split.avail_index = attr.used_index; 20871a86b377SEli Cohen return 0; 20881a86b377SEli Cohen } 20891a86b377SEli Cohen 20901a86b377SEli Cohen static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) 20911a86b377SEli Cohen { 20921a86b377SEli Cohen return PAGE_SIZE; 20931a86b377SEli Cohen } 20941a86b377SEli Cohen 2095d4821902SGautam Dawar static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdpa, u16 idx) 2096d4821902SGautam Dawar { 2097d4821902SGautam Dawar return 0; 2098d4821902SGautam Dawar } 2099d4821902SGautam Dawar 21001a86b377SEli Cohen enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9, 21011a86b377SEli Cohen MLX5_VIRTIO_NET_F_CSUM = 1 << 10, 21021a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11, 21031a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12, 21041a86b377SEli Cohen }; 21051a86b377SEli Cohen 21061a86b377SEli Cohen static u64 mlx_to_vritio_features(u16 dev_features) 21071a86b377SEli Cohen { 21081a86b377SEli Cohen u64 result = 0; 21091a86b377SEli Cohen 21101a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM) 2111cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM); 21121a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_CSUM) 2113cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_CSUM); 21141a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6) 2115cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6); 21161a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4) 2117cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4); 21181a86b377SEli Cohen 21191a86b377SEli Cohen return result; 21201a86b377SEli Cohen } 21211a86b377SEli Cohen 212279de65edSEli Cohen static u64 get_supported_features(struct mlx5_core_dev *mdev) 212379de65edSEli Cohen { 212479de65edSEli Cohen u64 mlx_vdpa_features = 0; 212579de65edSEli Cohen u16 dev_features; 212679de65edSEli Cohen 212779de65edSEli Cohen dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mdev, device_features_bits_mask); 212879de65edSEli Cohen mlx_vdpa_features |= mlx_to_vritio_features(dev_features); 212979de65edSEli Cohen if (MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_version_1_0)) 213079de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_F_VERSION_1); 213179de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM); 213279de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VQ); 213379de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR); 213479de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MQ); 213579de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_STATUS); 213679de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MTU); 2137*baf2ad3fSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VLAN); 213879de65edSEli Cohen 213979de65edSEli Cohen return mlx_vdpa_features; 214079de65edSEli Cohen } 214179de65edSEli Cohen 2142a64917bcSEli Cohen static u64 mlx5_vdpa_get_device_features(struct vdpa_device *vdev) 21431a86b377SEli Cohen { 21441a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 21451a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 21465262912eSEli Cohen 21471a86b377SEli Cohen print_features(mvdev, ndev->mvdev.mlx_features, false); 21481a86b377SEli Cohen return ndev->mvdev.mlx_features; 21491a86b377SEli Cohen } 21501a86b377SEli Cohen 215130c22f38SSi-Wei Liu static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features) 21521a86b377SEli Cohen { 215330c22f38SSi-Wei Liu /* Minimum features to expect */ 2154cbb52359SNathan Chancellor if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) 21551a86b377SEli Cohen return -EOPNOTSUPP; 21561a86b377SEli Cohen 215730c22f38SSi-Wei Liu /* Double check features combination sent down by the driver. 215830c22f38SSi-Wei Liu * Fail invalid features due to absence of the depended feature. 215930c22f38SSi-Wei Liu * 216030c22f38SSi-Wei Liu * Per VIRTIO v1.1 specification, section 5.1.3.1 Feature bit 216130c22f38SSi-Wei Liu * requirements: "VIRTIO_NET_F_MQ Requires VIRTIO_NET_F_CTRL_VQ". 216230c22f38SSi-Wei Liu * By failing the invalid features sent down by untrusted drivers, 216330c22f38SSi-Wei Liu * we're assured the assumption made upon is_index_valid() and 216430c22f38SSi-Wei Liu * is_ctrl_vq_idx() will not be compromised. 216530c22f38SSi-Wei Liu */ 216630c22f38SSi-Wei Liu if ((features & (BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) == 216730c22f38SSi-Wei Liu BIT_ULL(VIRTIO_NET_F_MQ)) 216830c22f38SSi-Wei Liu return -EINVAL; 216930c22f38SSi-Wei Liu 21701a86b377SEli Cohen return 0; 21711a86b377SEli Cohen } 21721a86b377SEli Cohen 2173ae0428deSEli Cohen static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev) 21741a86b377SEli Cohen { 2175ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 21765262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 21771a86b377SEli Cohen int err; 21781a86b377SEli Cohen int i; 21791a86b377SEli Cohen 2180acde3929SEli Cohen for (i = 0; i < mvdev->max_vqs; i++) { 21811a86b377SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 21821a86b377SEli Cohen if (err) 21831a86b377SEli Cohen goto err_vq; 21841a86b377SEli Cohen } 21851a86b377SEli Cohen 21865262912eSEli Cohen if (mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) { 21875262912eSEli Cohen err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features, 21885262912eSEli Cohen MLX5_CVQ_MAX_ENT, false, 21895262912eSEli Cohen (struct vring_desc *)(uintptr_t)cvq->desc_addr, 21905262912eSEli Cohen (struct vring_avail *)(uintptr_t)cvq->driver_addr, 21915262912eSEli Cohen (struct vring_used *)(uintptr_t)cvq->device_addr); 21925262912eSEli Cohen if (err) 21935262912eSEli Cohen goto err_vq; 21945262912eSEli Cohen } 21955262912eSEli Cohen 21961a86b377SEli Cohen return 0; 21971a86b377SEli Cohen 21981a86b377SEli Cohen err_vq: 21991a86b377SEli Cohen for (--i; i >= 0; i--) 22001a86b377SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 22011a86b377SEli Cohen 22021a86b377SEli Cohen return err; 22031a86b377SEli Cohen } 22041a86b377SEli Cohen 22051a86b377SEli Cohen static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) 22061a86b377SEli Cohen { 22071a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 22081a86b377SEli Cohen int i; 22091a86b377SEli Cohen 22101a86b377SEli Cohen for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { 22111a86b377SEli Cohen mvq = &ndev->vqs[i]; 22121a86b377SEli Cohen if (!mvq->initialized) 22131a86b377SEli Cohen continue; 22141a86b377SEli Cohen 22151a86b377SEli Cohen teardown_vq(ndev, mvq); 22161a86b377SEli Cohen } 22171a86b377SEli Cohen } 22181a86b377SEli Cohen 2219e4fc6650SEli Cohen static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) 22208a7c3213SMichael S. Tsirkin { 2221e4fc6650SEli Cohen if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_CTRL_VQ)) { 2222e4fc6650SEli Cohen if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) { 2223e4fc6650SEli Cohen /* MQ supported. CVQ index is right above the last data virtqueue's */ 2224e4fc6650SEli Cohen mvdev->max_idx = mvdev->max_vqs; 2225e4fc6650SEli Cohen } else { 2226e4fc6650SEli Cohen /* Only CVQ supportted. data virtqueues occupy indices 0 and 1. 2227e4fc6650SEli Cohen * CVQ gets index 2 2228e4fc6650SEli Cohen */ 2229e4fc6650SEli Cohen mvdev->max_idx = 2; 22308a7c3213SMichael S. Tsirkin } 2231e4fc6650SEli Cohen } else { 2232e4fc6650SEli Cohen /* Two data virtqueues only: one for rx and one for tx */ 2233e4fc6650SEli Cohen mvdev->max_idx = 1; 2234e4fc6650SEli Cohen } 223536bdcf31SEli Cohen } 223636bdcf31SEli Cohen 2237a64917bcSEli Cohen static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) 22381a86b377SEli Cohen { 22391a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22401a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 22411a86b377SEli Cohen int err; 22421a86b377SEli Cohen 22431a86b377SEli Cohen print_features(mvdev, features, true); 22441a86b377SEli Cohen 224530c22f38SSi-Wei Liu err = verify_driver_features(mvdev, features); 22461a86b377SEli Cohen if (err) 22471a86b377SEli Cohen return err; 22481a86b377SEli Cohen 22491a86b377SEli Cohen ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; 2250b03fc43eSEli Cohen if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)) 2251acde3929SEli Cohen ndev->rqt_size = mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs); 2252b03fc43eSEli Cohen else 2253acde3929SEli Cohen ndev->rqt_size = 1; 2254acde3929SEli Cohen 2255acde3929SEli Cohen ndev->cur_num_vqs = 2 * ndev->rqt_size; 2256b03fc43eSEli Cohen 2257e4fc6650SEli Cohen update_cvq_info(mvdev); 22581a86b377SEli Cohen return err; 22591a86b377SEli Cohen } 22601a86b377SEli Cohen 22611a86b377SEli Cohen static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 22621a86b377SEli Cohen { 2263edf747afSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2264edf747afSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2265edf747afSEli Cohen 2266edf747afSEli Cohen ndev->config_cb = *cb; 22671a86b377SEli Cohen } 22681a86b377SEli Cohen 22691a86b377SEli Cohen #define MLX5_VDPA_MAX_VQ_ENTRIES 256 22701a86b377SEli Cohen static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev) 22711a86b377SEli Cohen { 22721a86b377SEli Cohen return MLX5_VDPA_MAX_VQ_ENTRIES; 22731a86b377SEli Cohen } 22741a86b377SEli Cohen 22751a86b377SEli Cohen static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev) 22761a86b377SEli Cohen { 22771a86b377SEli Cohen return VIRTIO_ID_NET; 22781a86b377SEli Cohen } 22791a86b377SEli Cohen 22801a86b377SEli Cohen static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev) 22811a86b377SEli Cohen { 22821a86b377SEli Cohen return PCI_VENDOR_ID_MELLANOX; 22831a86b377SEli Cohen } 22841a86b377SEli Cohen 22851a86b377SEli Cohen static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev) 22861a86b377SEli Cohen { 22871a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22881a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 22891a86b377SEli Cohen 22901a86b377SEli Cohen print_status(mvdev, ndev->mvdev.status, false); 22911a86b377SEli Cohen return ndev->mvdev.status; 22921a86b377SEli Cohen } 22931a86b377SEli Cohen 22941a86b377SEli Cohen static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 22951a86b377SEli Cohen { 22961a86b377SEli Cohen struct mlx5_vq_restore_info *ri = &mvq->ri; 229752893733SEli Cohen struct mlx5_virtq_attr attr = {}; 22981a86b377SEli Cohen int err; 22991a86b377SEli Cohen 230052893733SEli Cohen if (mvq->initialized) { 23011a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 23021a86b377SEli Cohen if (err) 23031a86b377SEli Cohen return err; 230452893733SEli Cohen } 23051a86b377SEli Cohen 23061a86b377SEli Cohen ri->avail_index = attr.available_index; 2307b35ccebeSEli Cohen ri->used_index = attr.used_index; 23081a86b377SEli Cohen ri->ready = mvq->ready; 23091a86b377SEli Cohen ri->num_ent = mvq->num_ent; 23101a86b377SEli Cohen ri->desc_addr = mvq->desc_addr; 23111a86b377SEli Cohen ri->device_addr = mvq->device_addr; 23121a86b377SEli Cohen ri->driver_addr = mvq->driver_addr; 23131a86b377SEli Cohen ri->restore = true; 23141a86b377SEli Cohen return 0; 23151a86b377SEli Cohen } 23161a86b377SEli Cohen 23171a86b377SEli Cohen static int save_channels_info(struct mlx5_vdpa_net *ndev) 23181a86b377SEli Cohen { 23191a86b377SEli Cohen int i; 23201a86b377SEli Cohen 23211a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 23221a86b377SEli Cohen memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); 23231a86b377SEli Cohen save_channel_info(ndev, &ndev->vqs[i]); 23241a86b377SEli Cohen } 23251a86b377SEli Cohen return 0; 23261a86b377SEli Cohen } 23271a86b377SEli Cohen 23281a86b377SEli Cohen static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) 23291a86b377SEli Cohen { 23301a86b377SEli Cohen int i; 23311a86b377SEli Cohen 23321a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 23331a86b377SEli Cohen memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 23341a86b377SEli Cohen } 23351a86b377SEli Cohen 23361a86b377SEli Cohen static void restore_channels_info(struct mlx5_vdpa_net *ndev) 23371a86b377SEli Cohen { 23381a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 23391a86b377SEli Cohen struct mlx5_vq_restore_info *ri; 23401a86b377SEli Cohen int i; 23411a86b377SEli Cohen 23421a86b377SEli Cohen mlx5_clear_vqs(ndev); 23431a86b377SEli Cohen init_mvqs(ndev); 23441a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 23451a86b377SEli Cohen mvq = &ndev->vqs[i]; 23461a86b377SEli Cohen ri = &mvq->ri; 23471a86b377SEli Cohen if (!ri->restore) 23481a86b377SEli Cohen continue; 23491a86b377SEli Cohen 23501a86b377SEli Cohen mvq->avail_idx = ri->avail_index; 2351b35ccebeSEli Cohen mvq->used_idx = ri->used_index; 23521a86b377SEli Cohen mvq->ready = ri->ready; 23531a86b377SEli Cohen mvq->num_ent = ri->num_ent; 23541a86b377SEli Cohen mvq->desc_addr = ri->desc_addr; 23551a86b377SEli Cohen mvq->device_addr = ri->device_addr; 23561a86b377SEli Cohen mvq->driver_addr = ri->driver_addr; 23571a86b377SEli Cohen } 23581a86b377SEli Cohen } 23591a86b377SEli Cohen 2360ae0428deSEli Cohen static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) 23611a86b377SEli Cohen { 2362ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 23631a86b377SEli Cohen int err; 23641a86b377SEli Cohen 23651a86b377SEli Cohen suspend_vqs(ndev); 23661a86b377SEli Cohen err = save_channels_info(ndev); 23671a86b377SEli Cohen if (err) 23681a86b377SEli Cohen goto err_mr; 23691a86b377SEli Cohen 23701a86b377SEli Cohen teardown_driver(ndev); 2371ae0428deSEli Cohen mlx5_vdpa_destroy_mr(mvdev); 2372ae0428deSEli Cohen err = mlx5_vdpa_create_mr(mvdev, iotlb); 23731a86b377SEli Cohen if (err) 23741a86b377SEli Cohen goto err_mr; 23751a86b377SEli Cohen 2376ae0428deSEli Cohen if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 23771c80cf03SJason Wang goto err_mr; 23781897f0b6SEli Cohen 23791a86b377SEli Cohen restore_channels_info(ndev); 2380ae0428deSEli Cohen err = setup_driver(mvdev); 23811a86b377SEli Cohen if (err) 23821a86b377SEli Cohen goto err_setup; 23831a86b377SEli Cohen 23841a86b377SEli Cohen return 0; 23851a86b377SEli Cohen 23861a86b377SEli Cohen err_setup: 2387ae0428deSEli Cohen mlx5_vdpa_destroy_mr(mvdev); 23881a86b377SEli Cohen err_mr: 23891a86b377SEli Cohen return err; 23901a86b377SEli Cohen } 23911a86b377SEli Cohen 23921c80cf03SJason Wang /* reslock must be held for this function */ 2393ae0428deSEli Cohen static int setup_driver(struct mlx5_vdpa_dev *mvdev) 23941a86b377SEli Cohen { 2395ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 23961a86b377SEli Cohen int err; 23971a86b377SEli Cohen 2398759ae7f9SEli Cohen WARN_ON(!rwsem_is_locked(&ndev->reslock)); 23991c80cf03SJason Wang 24001a86b377SEli Cohen if (ndev->setup) { 2401ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n"); 24021a86b377SEli Cohen err = 0; 24031a86b377SEli Cohen goto out; 24041a86b377SEli Cohen } 2405ae0428deSEli Cohen err = setup_virtqueues(mvdev); 24061a86b377SEli Cohen if (err) { 2407ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "setup_virtqueues\n"); 24081a86b377SEli Cohen goto out; 24091a86b377SEli Cohen } 24101a86b377SEli Cohen 24111a86b377SEli Cohen err = create_rqt(ndev); 24121a86b377SEli Cohen if (err) { 2413ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "create_rqt\n"); 24141a86b377SEli Cohen goto err_rqt; 24151a86b377SEli Cohen } 24161a86b377SEli Cohen 24171a86b377SEli Cohen err = create_tir(ndev); 24181a86b377SEli Cohen if (err) { 2419ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "create_tir\n"); 24201a86b377SEli Cohen goto err_tir; 24211a86b377SEli Cohen } 24221a86b377SEli Cohen 2423*baf2ad3fSEli Cohen err = setup_steering(ndev); 24241a86b377SEli Cohen if (err) { 2425*baf2ad3fSEli Cohen mlx5_vdpa_warn(mvdev, "setup_steering\n"); 24261a86b377SEli Cohen goto err_fwd; 24271a86b377SEli Cohen } 24281a86b377SEli Cohen ndev->setup = true; 24291a86b377SEli Cohen 24301a86b377SEli Cohen return 0; 24311a86b377SEli Cohen 24321a86b377SEli Cohen err_fwd: 24331a86b377SEli Cohen destroy_tir(ndev); 24341a86b377SEli Cohen err_tir: 24351a86b377SEli Cohen destroy_rqt(ndev); 24361a86b377SEli Cohen err_rqt: 24371a86b377SEli Cohen teardown_virtqueues(ndev); 24381a86b377SEli Cohen out: 24391a86b377SEli Cohen return err; 24401a86b377SEli Cohen } 24411a86b377SEli Cohen 24421c80cf03SJason Wang /* reslock must be held for this function */ 24431a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev) 24441a86b377SEli Cohen { 24451c80cf03SJason Wang 2446759ae7f9SEli Cohen WARN_ON(!rwsem_is_locked(&ndev->reslock)); 24471c80cf03SJason Wang 24481a86b377SEli Cohen if (!ndev->setup) 24491c80cf03SJason Wang return; 24501a86b377SEli Cohen 2451*baf2ad3fSEli Cohen teardown_steering(ndev); 24521a86b377SEli Cohen destroy_tir(ndev); 24531a86b377SEli Cohen destroy_rqt(ndev); 24541a86b377SEli Cohen teardown_virtqueues(ndev); 24551a86b377SEli Cohen ndev->setup = false; 24561a86b377SEli Cohen } 24571a86b377SEli Cohen 2458e3aadf2eSEli Cohen static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) 2459e3aadf2eSEli Cohen { 2460e3aadf2eSEli Cohen int i; 2461e3aadf2eSEli Cohen 2462e3aadf2eSEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 2463e3aadf2eSEli Cohen ndev->vqs[i].ready = false; 2464ef12e4bfSEli Cohen 2465ef12e4bfSEli Cohen ndev->mvdev.cvq.ready = false; 2466e3aadf2eSEli Cohen } 2467e3aadf2eSEli Cohen 24681a86b377SEli Cohen static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) 24691a86b377SEli Cohen { 24701a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 24711a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 24721a86b377SEli Cohen int err; 24731a86b377SEli Cohen 24741a86b377SEli Cohen print_status(mvdev, status, true); 24751a86b377SEli Cohen 2476759ae7f9SEli Cohen down_write(&ndev->reslock); 24771c80cf03SJason Wang 24781a86b377SEli Cohen if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { 24791a86b377SEli Cohen if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 2480ae0428deSEli Cohen err = setup_driver(mvdev); 24811a86b377SEli Cohen if (err) { 24821a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); 24831a86b377SEli Cohen goto err_setup; 24841a86b377SEli Cohen } 24851a86b377SEli Cohen } else { 24861a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); 24871c80cf03SJason Wang goto err_clear; 24881a86b377SEli Cohen } 24891a86b377SEli Cohen } 24901a86b377SEli Cohen 24911a86b377SEli Cohen ndev->mvdev.status = status; 2492759ae7f9SEli Cohen up_write(&ndev->reslock); 24931a86b377SEli Cohen return; 24941a86b377SEli Cohen 24951a86b377SEli Cohen err_setup: 24961a86b377SEli Cohen mlx5_vdpa_destroy_mr(&ndev->mvdev); 24971a86b377SEli Cohen ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; 24981c80cf03SJason Wang err_clear: 2499759ae7f9SEli Cohen up_write(&ndev->reslock); 25001a86b377SEli Cohen } 25011a86b377SEli Cohen 25020686082dSXie Yongji static int mlx5_vdpa_reset(struct vdpa_device *vdev) 25030686082dSXie Yongji { 25040686082dSXie Yongji struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 25050686082dSXie Yongji struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 25060686082dSXie Yongji 25070686082dSXie Yongji print_status(mvdev, 0, true); 25080686082dSXie Yongji mlx5_vdpa_info(mvdev, "performing device reset\n"); 25091c80cf03SJason Wang 2510759ae7f9SEli Cohen down_write(&ndev->reslock); 25110686082dSXie Yongji teardown_driver(ndev); 25120686082dSXie Yongji clear_vqs_ready(ndev); 25130686082dSXie Yongji mlx5_vdpa_destroy_mr(&ndev->mvdev); 25140686082dSXie Yongji ndev->mvdev.status = 0; 2515b03fc43eSEli Cohen ndev->cur_num_vqs = 0; 25161892a3d4SEli Cohen ndev->mvdev.cvq.received_desc = 0; 25171892a3d4SEli Cohen ndev->mvdev.cvq.completed_desc = 0; 251875560522SEli Cohen memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1)); 25190686082dSXie Yongji ndev->mvdev.actual_features = 0; 25200686082dSXie Yongji ++mvdev->generation; 25210686082dSXie Yongji if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 25220686082dSXie Yongji if (mlx5_vdpa_create_mr(mvdev, NULL)) 25230686082dSXie Yongji mlx5_vdpa_warn(mvdev, "create MR failed\n"); 25240686082dSXie Yongji } 2525759ae7f9SEli Cohen up_write(&ndev->reslock); 25260686082dSXie Yongji 25270686082dSXie Yongji return 0; 25280686082dSXie Yongji } 25290686082dSXie Yongji 2530442706f9SStefano Garzarella static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev) 2531442706f9SStefano Garzarella { 2532442706f9SStefano Garzarella return sizeof(struct virtio_net_config); 2533442706f9SStefano Garzarella } 2534442706f9SStefano Garzarella 25351a86b377SEli Cohen static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, 25361a86b377SEli Cohen unsigned int len) 25371a86b377SEli Cohen { 25381a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 25391a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 25401a86b377SEli Cohen 2541dcfde163SStefano Garzarella if (offset + len <= sizeof(struct virtio_net_config)) 25422874211fSDan Carpenter memcpy(buf, (u8 *)&ndev->config + offset, len); 25431a86b377SEli Cohen } 25441a86b377SEli Cohen 25451a86b377SEli Cohen static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf, 25461a86b377SEli Cohen unsigned int len) 25471a86b377SEli Cohen { 25481a86b377SEli Cohen /* not supported */ 25491a86b377SEli Cohen } 25501a86b377SEli Cohen 25511a86b377SEli Cohen static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) 25521a86b377SEli Cohen { 25531a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 25541a86b377SEli Cohen 25551a86b377SEli Cohen return mvdev->generation; 25561a86b377SEli Cohen } 25571a86b377SEli Cohen 2558db9adcbfSGautam Dawar static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid, 2559db9adcbfSGautam Dawar struct vhost_iotlb *iotlb) 25601a86b377SEli Cohen { 25611a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 25621c80cf03SJason Wang struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 25631a86b377SEli Cohen bool change_map; 25641a86b377SEli Cohen int err; 25651a86b377SEli Cohen 2566759ae7f9SEli Cohen down_write(&ndev->reslock); 25671c80cf03SJason Wang 25681a86b377SEli Cohen err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map); 25691a86b377SEli Cohen if (err) { 25701a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); 25711c80cf03SJason Wang goto err; 25721a86b377SEli Cohen } 25731a86b377SEli Cohen 25741a86b377SEli Cohen if (change_map) 25751c80cf03SJason Wang err = mlx5_vdpa_change_map(mvdev, iotlb); 25761a86b377SEli Cohen 25771c80cf03SJason Wang err: 2578759ae7f9SEli Cohen up_write(&ndev->reslock); 25791c80cf03SJason Wang return err; 25801a86b377SEli Cohen } 25811a86b377SEli Cohen 25821a86b377SEli Cohen static void mlx5_vdpa_free(struct vdpa_device *vdev) 25831a86b377SEli Cohen { 25841a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 25857c9f131fSEli Cohen struct mlx5_core_dev *pfmdev; 25861a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 25871a86b377SEli Cohen 25881a86b377SEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 25891a86b377SEli Cohen 25901a86b377SEli Cohen free_resources(ndev); 25916f5312f8SEli Cohen mlx5_vdpa_destroy_mr(mvdev); 25927c9f131fSEli Cohen if (!is_zero_ether_addr(ndev->config.mac)) { 25937c9f131fSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 25947c9f131fSEli Cohen mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); 25957c9f131fSEli Cohen } 25961a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 259775560522SEli Cohen kfree(ndev->event_cbs); 259875560522SEli Cohen kfree(ndev->vqs); 25991a86b377SEli Cohen } 26001a86b377SEli Cohen 26011a86b377SEli Cohen static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx) 26021a86b377SEli Cohen { 2603b57c46cbSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 26041a86b377SEli Cohen struct vdpa_notification_area ret = {}; 2605b57c46cbSEli Cohen struct mlx5_vdpa_net *ndev; 2606b57c46cbSEli Cohen phys_addr_t addr; 26071a86b377SEli Cohen 26085262912eSEli Cohen if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2609e4fc6650SEli Cohen return ret; 2610e4fc6650SEli Cohen 2611b57c46cbSEli Cohen /* If SF BAR size is smaller than PAGE_SIZE, do not use direct 2612b57c46cbSEli Cohen * notification to avoid the risk of mapping pages that contain BAR of more 2613b57c46cbSEli Cohen * than one SF 2614b57c46cbSEli Cohen */ 2615b57c46cbSEli Cohen if (MLX5_CAP_GEN(mvdev->mdev, log_min_sf_size) + 12 < PAGE_SHIFT) 2616b57c46cbSEli Cohen return ret; 2617b57c46cbSEli Cohen 2618b57c46cbSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 2619b57c46cbSEli Cohen addr = (phys_addr_t)ndev->mvdev.res.phys_kick_addr; 2620b57c46cbSEli Cohen ret.addr = addr; 2621b57c46cbSEli Cohen ret.size = PAGE_SIZE; 26221a86b377SEli Cohen return ret; 26231a86b377SEli Cohen } 26241a86b377SEli Cohen 26251a86b377SEli Cohen static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) 26261a86b377SEli Cohen { 26271a86b377SEli Cohen return -EOPNOTSUPP; 26281a86b377SEli Cohen } 26291a86b377SEli Cohen 2630a64917bcSEli Cohen static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) 2631a64917bcSEli Cohen { 2632a64917bcSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2633a64917bcSEli Cohen 2634a64917bcSEli Cohen return mvdev->actual_features; 2635a64917bcSEli Cohen } 2636a64917bcSEli Cohen 26371892a3d4SEli Cohen static int counter_set_query(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 26381892a3d4SEli Cohen u64 *received_desc, u64 *completed_desc) 26391892a3d4SEli Cohen { 26401892a3d4SEli Cohen u32 in[MLX5_ST_SZ_DW(query_virtio_q_counters_in)] = {}; 26411892a3d4SEli Cohen u32 out[MLX5_ST_SZ_DW(query_virtio_q_counters_out)] = {}; 26421892a3d4SEli Cohen void *cmd_hdr; 26431892a3d4SEli Cohen void *ctx; 26441892a3d4SEli Cohen int err; 26451892a3d4SEli Cohen 26461892a3d4SEli Cohen if (!counters_supported(&ndev->mvdev)) 26471892a3d4SEli Cohen return -EOPNOTSUPP; 26481892a3d4SEli Cohen 26491892a3d4SEli Cohen if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 26501892a3d4SEli Cohen return -EAGAIN; 26511892a3d4SEli Cohen 26521892a3d4SEli Cohen cmd_hdr = MLX5_ADDR_OF(query_virtio_q_counters_in, in, hdr); 26531892a3d4SEli Cohen 26541892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 26551892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 26561892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 26571892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->counter_set_id); 26581892a3d4SEli Cohen 26591892a3d4SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 26601892a3d4SEli Cohen if (err) 26611892a3d4SEli Cohen return err; 26621892a3d4SEli Cohen 26631892a3d4SEli Cohen ctx = MLX5_ADDR_OF(query_virtio_q_counters_out, out, counters); 26641892a3d4SEli Cohen *received_desc = MLX5_GET64(virtio_q_counters, ctx, received_desc); 26651892a3d4SEli Cohen *completed_desc = MLX5_GET64(virtio_q_counters, ctx, completed_desc); 26661892a3d4SEli Cohen return 0; 26671892a3d4SEli Cohen } 26681892a3d4SEli Cohen 26691892a3d4SEli Cohen static int mlx5_vdpa_get_vendor_vq_stats(struct vdpa_device *vdev, u16 idx, 26701892a3d4SEli Cohen struct sk_buff *msg, 26711892a3d4SEli Cohen struct netlink_ext_ack *extack) 26721892a3d4SEli Cohen { 26731892a3d4SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 26741892a3d4SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 26751892a3d4SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 26761892a3d4SEli Cohen struct mlx5_control_vq *cvq; 26771892a3d4SEli Cohen u64 received_desc; 26781892a3d4SEli Cohen u64 completed_desc; 26791892a3d4SEli Cohen int err = 0; 26801892a3d4SEli Cohen 2681759ae7f9SEli Cohen down_read(&ndev->reslock); 26821892a3d4SEli Cohen if (!is_index_valid(mvdev, idx)) { 26831892a3d4SEli Cohen NL_SET_ERR_MSG_MOD(extack, "virtqueue index is not valid"); 26841892a3d4SEli Cohen err = -EINVAL; 26851892a3d4SEli Cohen goto out_err; 26861892a3d4SEli Cohen } 26871892a3d4SEli Cohen 26881892a3d4SEli Cohen if (idx == ctrl_vq_idx(mvdev)) { 26891892a3d4SEli Cohen cvq = &mvdev->cvq; 26901892a3d4SEli Cohen received_desc = cvq->received_desc; 26911892a3d4SEli Cohen completed_desc = cvq->completed_desc; 26921892a3d4SEli Cohen goto out; 26931892a3d4SEli Cohen } 26941892a3d4SEli Cohen 26951892a3d4SEli Cohen mvq = &ndev->vqs[idx]; 26961892a3d4SEli Cohen err = counter_set_query(ndev, mvq, &received_desc, &completed_desc); 26971892a3d4SEli Cohen if (err) { 26981892a3d4SEli Cohen NL_SET_ERR_MSG_MOD(extack, "failed to query hardware"); 26991892a3d4SEli Cohen goto out_err; 27001892a3d4SEli Cohen } 27011892a3d4SEli Cohen 27021892a3d4SEli Cohen out: 27031892a3d4SEli Cohen err = -EMSGSIZE; 27041892a3d4SEli Cohen if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "received_desc")) 27051892a3d4SEli Cohen goto out_err; 27061892a3d4SEli Cohen 27071892a3d4SEli Cohen if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, received_desc, 27081892a3d4SEli Cohen VDPA_ATTR_PAD)) 27091892a3d4SEli Cohen goto out_err; 27101892a3d4SEli Cohen 27111892a3d4SEli Cohen if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "completed_desc")) 27121892a3d4SEli Cohen goto out_err; 27131892a3d4SEli Cohen 27141892a3d4SEli Cohen if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, completed_desc, 27151892a3d4SEli Cohen VDPA_ATTR_PAD)) 27161892a3d4SEli Cohen goto out_err; 27171892a3d4SEli Cohen 27181892a3d4SEli Cohen err = 0; 27191892a3d4SEli Cohen out_err: 2720759ae7f9SEli Cohen up_read(&ndev->reslock); 27211892a3d4SEli Cohen return err; 27221892a3d4SEli Cohen } 27231892a3d4SEli Cohen 27241a86b377SEli Cohen static const struct vdpa_config_ops mlx5_vdpa_ops = { 27251a86b377SEli Cohen .set_vq_address = mlx5_vdpa_set_vq_address, 27261a86b377SEli Cohen .set_vq_num = mlx5_vdpa_set_vq_num, 27271a86b377SEli Cohen .kick_vq = mlx5_vdpa_kick_vq, 27281a86b377SEli Cohen .set_vq_cb = mlx5_vdpa_set_vq_cb, 27291a86b377SEli Cohen .set_vq_ready = mlx5_vdpa_set_vq_ready, 27301a86b377SEli Cohen .get_vq_ready = mlx5_vdpa_get_vq_ready, 27311a86b377SEli Cohen .set_vq_state = mlx5_vdpa_set_vq_state, 27321a86b377SEli Cohen .get_vq_state = mlx5_vdpa_get_vq_state, 27331892a3d4SEli Cohen .get_vendor_vq_stats = mlx5_vdpa_get_vendor_vq_stats, 27341a86b377SEli Cohen .get_vq_notification = mlx5_get_vq_notification, 27351a86b377SEli Cohen .get_vq_irq = mlx5_get_vq_irq, 27361a86b377SEli Cohen .get_vq_align = mlx5_vdpa_get_vq_align, 2737d4821902SGautam Dawar .get_vq_group = mlx5_vdpa_get_vq_group, 2738a64917bcSEli Cohen .get_device_features = mlx5_vdpa_get_device_features, 2739a64917bcSEli Cohen .set_driver_features = mlx5_vdpa_set_driver_features, 2740a64917bcSEli Cohen .get_driver_features = mlx5_vdpa_get_driver_features, 27411a86b377SEli Cohen .set_config_cb = mlx5_vdpa_set_config_cb, 27421a86b377SEli Cohen .get_vq_num_max = mlx5_vdpa_get_vq_num_max, 27431a86b377SEli Cohen .get_device_id = mlx5_vdpa_get_device_id, 27441a86b377SEli Cohen .get_vendor_id = mlx5_vdpa_get_vendor_id, 27451a86b377SEli Cohen .get_status = mlx5_vdpa_get_status, 27461a86b377SEli Cohen .set_status = mlx5_vdpa_set_status, 27470686082dSXie Yongji .reset = mlx5_vdpa_reset, 2748442706f9SStefano Garzarella .get_config_size = mlx5_vdpa_get_config_size, 27491a86b377SEli Cohen .get_config = mlx5_vdpa_get_config, 27501a86b377SEli Cohen .set_config = mlx5_vdpa_set_config, 27511a86b377SEli Cohen .get_generation = mlx5_vdpa_get_generation, 27521a86b377SEli Cohen .set_map = mlx5_vdpa_set_map, 27531a86b377SEli Cohen .free = mlx5_vdpa_free, 27541a86b377SEli Cohen }; 27551a86b377SEli Cohen 2756d084d996SSi-Wei Liu static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) 2757d084d996SSi-Wei Liu { 2758d084d996SSi-Wei Liu u16 hw_mtu; 2759d084d996SSi-Wei Liu int err; 2760d084d996SSi-Wei Liu 2761d084d996SSi-Wei Liu err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2762d084d996SSi-Wei Liu if (err) 2763d084d996SSi-Wei Liu return err; 2764d084d996SSi-Wei Liu 2765d084d996SSi-Wei Liu *mtu = hw_mtu - MLX5V_ETH_HARD_MTU; 2766d084d996SSi-Wei Liu return 0; 2767d084d996SSi-Wei Liu } 2768d084d996SSi-Wei Liu 27691a86b377SEli Cohen static int alloc_resources(struct mlx5_vdpa_net *ndev) 27701a86b377SEli Cohen { 27711a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 27721a86b377SEli Cohen int err; 27731a86b377SEli Cohen 27741a86b377SEli Cohen if (res->valid) { 27751a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); 27761a86b377SEli Cohen return -EEXIST; 27771a86b377SEli Cohen } 27781a86b377SEli Cohen 27791a86b377SEli Cohen err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); 27801a86b377SEli Cohen if (err) 27811a86b377SEli Cohen return err; 27821a86b377SEli Cohen 27831a86b377SEli Cohen err = create_tis(ndev); 27841a86b377SEli Cohen if (err) 27851a86b377SEli Cohen goto err_tis; 27861a86b377SEli Cohen 27871a86b377SEli Cohen res->valid = true; 27881a86b377SEli Cohen 27891a86b377SEli Cohen return 0; 27901a86b377SEli Cohen 27911a86b377SEli Cohen err_tis: 27921a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 27931a86b377SEli Cohen return err; 27941a86b377SEli Cohen } 27951a86b377SEli Cohen 27961a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev) 27971a86b377SEli Cohen { 27981a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 27991a86b377SEli Cohen 28001a86b377SEli Cohen if (!res->valid) 28011a86b377SEli Cohen return; 28021a86b377SEli Cohen 28031a86b377SEli Cohen destroy_tis(ndev); 28041a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 28051a86b377SEli Cohen res->valid = false; 28061a86b377SEli Cohen } 28071a86b377SEli Cohen 28081a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev) 28091a86b377SEli Cohen { 28101a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 28111a86b377SEli Cohen int i; 28121a86b377SEli Cohen 2813acde3929SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; ++i) { 28141a86b377SEli Cohen mvq = &ndev->vqs[i]; 28151a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 28161a86b377SEli Cohen mvq->index = i; 28171a86b377SEli Cohen mvq->ndev = ndev; 28181a86b377SEli Cohen mvq->fwqp.fw = true; 28191a86b377SEli Cohen } 28201a86b377SEli Cohen for (; i < ndev->mvdev.max_vqs; i++) { 28211a86b377SEli Cohen mvq = &ndev->vqs[i]; 28221a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 28231a86b377SEli Cohen mvq->index = i; 28241a86b377SEli Cohen mvq->ndev = ndev; 28251a86b377SEli Cohen } 28261a86b377SEli Cohen } 28271a86b377SEli Cohen 282858926c8aSEli Cohen struct mlx5_vdpa_mgmtdev { 282958926c8aSEli Cohen struct vdpa_mgmt_dev mgtdev; 283058926c8aSEli Cohen struct mlx5_adev *madev; 283158926c8aSEli Cohen struct mlx5_vdpa_net *ndev; 283258926c8aSEli Cohen }; 283358926c8aSEli Cohen 2834edf747afSEli Cohen static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) 2835edf747afSEli Cohen { 2836edf747afSEli Cohen u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; 2837edf747afSEli Cohen u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; 2838edf747afSEli Cohen int err; 2839edf747afSEli Cohen 2840edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE); 2841edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, op_mod, opmod); 2842edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, vport_number, vport); 2843edf747afSEli Cohen if (vport) 2844edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, other_vport, 1); 2845edf747afSEli Cohen 2846edf747afSEli Cohen err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out); 2847edf747afSEli Cohen if (err) 2848edf747afSEli Cohen return 0; 2849edf747afSEli Cohen 2850edf747afSEli Cohen return MLX5_GET(query_vport_state_out, out, state); 2851edf747afSEli Cohen } 2852edf747afSEli Cohen 2853edf747afSEli Cohen static bool get_link_state(struct mlx5_vdpa_dev *mvdev) 2854edf747afSEli Cohen { 2855edf747afSEli Cohen if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) == 2856edf747afSEli Cohen VPORT_STATE_UP) 2857edf747afSEli Cohen return true; 2858edf747afSEli Cohen 2859edf747afSEli Cohen return false; 2860edf747afSEli Cohen } 2861edf747afSEli Cohen 2862edf747afSEli Cohen static void update_carrier(struct work_struct *work) 2863edf747afSEli Cohen { 2864edf747afSEli Cohen struct mlx5_vdpa_wq_ent *wqent; 2865edf747afSEli Cohen struct mlx5_vdpa_dev *mvdev; 2866edf747afSEli Cohen struct mlx5_vdpa_net *ndev; 2867edf747afSEli Cohen 2868edf747afSEli Cohen wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 2869edf747afSEli Cohen mvdev = wqent->mvdev; 2870edf747afSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 2871edf747afSEli Cohen if (get_link_state(mvdev)) 2872edf747afSEli Cohen ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 2873edf747afSEli Cohen else 2874edf747afSEli Cohen ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 2875edf747afSEli Cohen 2876edf747afSEli Cohen if (ndev->config_cb.callback) 2877edf747afSEli Cohen ndev->config_cb.callback(ndev->config_cb.private); 2878edf747afSEli Cohen 2879edf747afSEli Cohen kfree(wqent); 2880edf747afSEli Cohen } 2881edf747afSEli Cohen 2882edf747afSEli Cohen static int event_handler(struct notifier_block *nb, unsigned long event, void *param) 2883edf747afSEli Cohen { 2884edf747afSEli Cohen struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb); 2885edf747afSEli Cohen struct mlx5_eqe *eqe = param; 2886edf747afSEli Cohen int ret = NOTIFY_DONE; 2887edf747afSEli Cohen struct mlx5_vdpa_wq_ent *wqent; 2888edf747afSEli Cohen 2889edf747afSEli Cohen if (event == MLX5_EVENT_TYPE_PORT_CHANGE) { 2890edf747afSEli Cohen switch (eqe->sub_type) { 2891edf747afSEli Cohen case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 2892edf747afSEli Cohen case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 2893edf747afSEli Cohen wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); 2894edf747afSEli Cohen if (!wqent) 2895edf747afSEli Cohen return NOTIFY_DONE; 2896edf747afSEli Cohen 2897edf747afSEli Cohen wqent->mvdev = &ndev->mvdev; 2898edf747afSEli Cohen INIT_WORK(&wqent->work, update_carrier); 2899edf747afSEli Cohen queue_work(ndev->mvdev.wq, &wqent->work); 2900edf747afSEli Cohen ret = NOTIFY_OK; 2901edf747afSEli Cohen break; 2902edf747afSEli Cohen default: 2903edf747afSEli Cohen return NOTIFY_DONE; 2904edf747afSEli Cohen } 2905edf747afSEli Cohen return ret; 2906edf747afSEli Cohen } 2907edf747afSEli Cohen return ret; 2908edf747afSEli Cohen } 2909edf747afSEli Cohen 29101e00e821SEli Cohen static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu) 29111e00e821SEli Cohen { 29121e00e821SEli Cohen int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); 29131e00e821SEli Cohen void *in; 29141e00e821SEli Cohen int err; 29151e00e821SEli Cohen 29161e00e821SEli Cohen in = kvzalloc(inlen, GFP_KERNEL); 29171e00e821SEli Cohen if (!in) 29181e00e821SEli Cohen return -ENOMEM; 29191e00e821SEli Cohen 29201e00e821SEli Cohen MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1); 29211e00e821SEli Cohen MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu, 29221e00e821SEli Cohen mtu + MLX5V_ETH_HARD_MTU); 29231e00e821SEli Cohen MLX5_SET(modify_nic_vport_context_in, in, opcode, 29241e00e821SEli Cohen MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); 29251e00e821SEli Cohen 29261e00e821SEli Cohen err = mlx5_cmd_exec_in(mdev, modify_nic_vport_context, in); 29271e00e821SEli Cohen 29281e00e821SEli Cohen kvfree(in); 29291e00e821SEli Cohen return err; 29301e00e821SEli Cohen } 29311e00e821SEli Cohen 2932d8ca2fa5SParav Pandit static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, 2933d8ca2fa5SParav Pandit const struct vdpa_dev_set_config *add_config) 29341a86b377SEli Cohen { 293558926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 29361a86b377SEli Cohen struct virtio_net_config *config; 29377c9f131fSEli Cohen struct mlx5_core_dev *pfmdev; 29381a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev; 29391a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 294058926c8aSEli Cohen struct mlx5_core_dev *mdev; 29411a86b377SEli Cohen u32 max_vqs; 2942246fd1caSEli Cohen u16 mtu; 29431a86b377SEli Cohen int err; 29441a86b377SEli Cohen 294558926c8aSEli Cohen if (mgtdev->ndev) 294658926c8aSEli Cohen return -ENOSPC; 294758926c8aSEli Cohen 294858926c8aSEli Cohen mdev = mgtdev->madev->mdev; 2949879753c8SEli Cohen if (!(MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_queue_type) & 2950879753c8SEli Cohen MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)) { 2951879753c8SEli Cohen dev_warn(mdev->device, "missing support for split virtqueues\n"); 2952879753c8SEli Cohen return -EOPNOTSUPP; 2953879753c8SEli Cohen } 2954879753c8SEli Cohen 2955acde3929SEli Cohen max_vqs = min_t(int, MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues), 2956acde3929SEli Cohen 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size)); 295775560522SEli Cohen if (max_vqs < 2) { 295875560522SEli Cohen dev_warn(mdev->device, 295975560522SEli Cohen "%d virtqueues are supported. At least 2 are required\n", 296075560522SEli Cohen max_vqs); 296175560522SEli Cohen return -EAGAIN; 296275560522SEli Cohen } 296375560522SEli Cohen 296475560522SEli Cohen if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP)) { 296575560522SEli Cohen if (add_config->net.max_vq_pairs > max_vqs / 2) 296675560522SEli Cohen return -EINVAL; 296775560522SEli Cohen max_vqs = min_t(u32, max_vqs, 2 * add_config->net.max_vq_pairs); 296875560522SEli Cohen } else { 296975560522SEli Cohen max_vqs = 2; 297075560522SEli Cohen } 29711a86b377SEli Cohen 29721a86b377SEli Cohen ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, 2973db9adcbfSGautam Dawar 1, 1, name, false); 29741a86b377SEli Cohen if (IS_ERR(ndev)) 297574c9729dSLeon Romanovsky return PTR_ERR(ndev); 29761a86b377SEli Cohen 297779de65edSEli Cohen ndev->mvdev.mlx_features = mgtdev->mgtdev.supported_features; 29781a86b377SEli Cohen ndev->mvdev.max_vqs = max_vqs; 29791a86b377SEli Cohen mvdev = &ndev->mvdev; 29801a86b377SEli Cohen mvdev->mdev = mdev; 298175560522SEli Cohen 298275560522SEli Cohen ndev->vqs = kcalloc(max_vqs, sizeof(*ndev->vqs), GFP_KERNEL); 298375560522SEli Cohen ndev->event_cbs = kcalloc(max_vqs + 1, sizeof(*ndev->event_cbs), GFP_KERNEL); 298475560522SEli Cohen if (!ndev->vqs || !ndev->event_cbs) { 298575560522SEli Cohen err = -ENOMEM; 298675560522SEli Cohen goto err_alloc; 298775560522SEli Cohen } 298875560522SEli Cohen 29891a86b377SEli Cohen init_mvqs(ndev); 2990759ae7f9SEli Cohen init_rwsem(&ndev->reslock); 29911a86b377SEli Cohen config = &ndev->config; 29921e00e821SEli Cohen 29931e00e821SEli Cohen if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU)) { 29941e00e821SEli Cohen err = config_func_mtu(mdev, add_config->net.mtu); 29951e00e821SEli Cohen if (err) 2996759ae7f9SEli Cohen goto err_alloc; 29971e00e821SEli Cohen } 29981e00e821SEli Cohen 2999246fd1caSEli Cohen err = query_mtu(mdev, &mtu); 30001a86b377SEli Cohen if (err) 3001759ae7f9SEli Cohen goto err_alloc; 30021a86b377SEli Cohen 3003246fd1caSEli Cohen ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, mtu); 30041a86b377SEli Cohen 3005edf747afSEli Cohen if (get_link_state(mvdev)) 3006edf747afSEli Cohen ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 3007edf747afSEli Cohen else 3008edf747afSEli Cohen ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 3009edf747afSEli Cohen 3010a007d940SEli Cohen if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) { 3011a007d940SEli Cohen memcpy(ndev->config.mac, add_config->net.mac, ETH_ALEN); 3012a007d940SEli Cohen } else { 30131a86b377SEli Cohen err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac); 30141a86b377SEli Cohen if (err) 3015759ae7f9SEli Cohen goto err_alloc; 3016a007d940SEli Cohen } 30171a86b377SEli Cohen 30187c9f131fSEli Cohen if (!is_zero_ether_addr(config->mac)) { 30197c9f131fSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev)); 30207c9f131fSEli Cohen err = mlx5_mpfs_add_mac(pfmdev, config->mac); 30217c9f131fSEli Cohen if (err) 3022759ae7f9SEli Cohen goto err_alloc; 30235262912eSEli Cohen 30245262912eSEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MAC); 30257c9f131fSEli Cohen } 30267c9f131fSEli Cohen 3027acde3929SEli Cohen config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, max_vqs / 2); 30287d23dcdfSEli Cohen mvdev->vdev.dma_dev = &mdev->pdev->dev; 30291a86b377SEli Cohen err = mlx5_vdpa_alloc_resources(&ndev->mvdev); 30301a86b377SEli Cohen if (err) 30317c9f131fSEli Cohen goto err_mpfs; 30321a86b377SEli Cohen 30336f5312f8SEli Cohen if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 30346f5312f8SEli Cohen err = mlx5_vdpa_create_mr(mvdev, NULL); 30351a86b377SEli Cohen if (err) 30361a86b377SEli Cohen goto err_res; 30376f5312f8SEli Cohen } 30386f5312f8SEli Cohen 30396f5312f8SEli Cohen err = alloc_resources(ndev); 30406f5312f8SEli Cohen if (err) 30416f5312f8SEli Cohen goto err_mr; 30421a86b377SEli Cohen 304355ebf0d6SJason Wang ndev->cvq_ent.mvdev = mvdev; 304455ebf0d6SJason Wang INIT_WORK(&ndev->cvq_ent.work, mlx5_cvq_kick_handler); 3045218bdd20SEli Cohen mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_wq"); 30465262912eSEli Cohen if (!mvdev->wq) { 30475262912eSEli Cohen err = -ENOMEM; 30485262912eSEli Cohen goto err_res2; 30495262912eSEli Cohen } 30505262912eSEli Cohen 3051edf747afSEli Cohen ndev->nb.notifier_call = event_handler; 3052edf747afSEli Cohen mlx5_notifier_register(mdev, &ndev->nb); 305358926c8aSEli Cohen mvdev->vdev.mdev = &mgtdev->mgtdev; 3054acde3929SEli Cohen err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1); 30551a86b377SEli Cohen if (err) 30561a86b377SEli Cohen goto err_reg; 30571a86b377SEli Cohen 305858926c8aSEli Cohen mgtdev->ndev = ndev; 305974c9729dSLeon Romanovsky return 0; 30601a86b377SEli Cohen 30611a86b377SEli Cohen err_reg: 30625262912eSEli Cohen destroy_workqueue(mvdev->wq); 30635262912eSEli Cohen err_res2: 30641a86b377SEli Cohen free_resources(ndev); 30656f5312f8SEli Cohen err_mr: 30666f5312f8SEli Cohen mlx5_vdpa_destroy_mr(mvdev); 30671a86b377SEli Cohen err_res: 30681a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 30697c9f131fSEli Cohen err_mpfs: 30707c9f131fSEli Cohen if (!is_zero_ether_addr(config->mac)) 30717c9f131fSEli Cohen mlx5_mpfs_del_mac(pfmdev, config->mac); 307275560522SEli Cohen err_alloc: 30731a86b377SEli Cohen put_device(&mvdev->vdev.dev); 307474c9729dSLeon Romanovsky return err; 30751a86b377SEli Cohen } 30761a86b377SEli Cohen 307758926c8aSEli Cohen static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev) 307858926c8aSEli Cohen { 307958926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 30805262912eSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(dev); 3081edf747afSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 3082ad6dc1daSEli Cohen struct workqueue_struct *wq; 308358926c8aSEli Cohen 3084edf747afSEli Cohen mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 3085ad6dc1daSEli Cohen wq = mvdev->wq; 3086ad6dc1daSEli Cohen mvdev->wq = NULL; 3087ad6dc1daSEli Cohen destroy_workqueue(wq); 308858926c8aSEli Cohen _vdpa_unregister_device(dev); 308958926c8aSEli Cohen mgtdev->ndev = NULL; 309058926c8aSEli Cohen } 309158926c8aSEli Cohen 309258926c8aSEli Cohen static const struct vdpa_mgmtdev_ops mdev_ops = { 309358926c8aSEli Cohen .dev_add = mlx5_vdpa_dev_add, 309458926c8aSEli Cohen .dev_del = mlx5_vdpa_dev_del, 309558926c8aSEli Cohen }; 309658926c8aSEli Cohen 309758926c8aSEli Cohen static struct virtio_device_id id_table[] = { 309858926c8aSEli Cohen { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 309958926c8aSEli Cohen { 0 }, 310058926c8aSEli Cohen }; 310158926c8aSEli Cohen 310258926c8aSEli Cohen static int mlx5v_probe(struct auxiliary_device *adev, 310358926c8aSEli Cohen const struct auxiliary_device_id *id) 310458926c8aSEli Cohen 310558926c8aSEli Cohen { 310658926c8aSEli Cohen struct mlx5_adev *madev = container_of(adev, struct mlx5_adev, adev); 310758926c8aSEli Cohen struct mlx5_core_dev *mdev = madev->mdev; 310858926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev; 310958926c8aSEli Cohen int err; 311058926c8aSEli Cohen 311158926c8aSEli Cohen mgtdev = kzalloc(sizeof(*mgtdev), GFP_KERNEL); 311258926c8aSEli Cohen if (!mgtdev) 311358926c8aSEli Cohen return -ENOMEM; 311458926c8aSEli Cohen 311558926c8aSEli Cohen mgtdev->mgtdev.ops = &mdev_ops; 311658926c8aSEli Cohen mgtdev->mgtdev.device = mdev->device; 311758926c8aSEli Cohen mgtdev->mgtdev.id_table = id_table; 311875560522SEli Cohen mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | 31191e00e821SEli Cohen BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP) | 31201e00e821SEli Cohen BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU); 312179de65edSEli Cohen mgtdev->mgtdev.max_supported_vqs = 312279de65edSEli Cohen MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1; 312379de65edSEli Cohen mgtdev->mgtdev.supported_features = get_supported_features(mdev); 312458926c8aSEli Cohen mgtdev->madev = madev; 312558926c8aSEli Cohen 312658926c8aSEli Cohen err = vdpa_mgmtdev_register(&mgtdev->mgtdev); 312758926c8aSEli Cohen if (err) 312858926c8aSEli Cohen goto reg_err; 312958926c8aSEli Cohen 313045e3a279SDavid E. Box auxiliary_set_drvdata(adev, mgtdev); 313158926c8aSEli Cohen 313258926c8aSEli Cohen return 0; 313358926c8aSEli Cohen 313458926c8aSEli Cohen reg_err: 313558926c8aSEli Cohen kfree(mgtdev); 313658926c8aSEli Cohen return err; 313758926c8aSEli Cohen } 313858926c8aSEli Cohen 313974c9729dSLeon Romanovsky static void mlx5v_remove(struct auxiliary_device *adev) 31401a86b377SEli Cohen { 314158926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev; 314274c9729dSLeon Romanovsky 314345e3a279SDavid E. Box mgtdev = auxiliary_get_drvdata(adev); 314458926c8aSEli Cohen vdpa_mgmtdev_unregister(&mgtdev->mgtdev); 314558926c8aSEli Cohen kfree(mgtdev); 31461a86b377SEli Cohen } 314774c9729dSLeon Romanovsky 314874c9729dSLeon Romanovsky static const struct auxiliary_device_id mlx5v_id_table[] = { 314974c9729dSLeon Romanovsky { .name = MLX5_ADEV_NAME ".vnet", }, 315074c9729dSLeon Romanovsky {}, 315174c9729dSLeon Romanovsky }; 315274c9729dSLeon Romanovsky 315374c9729dSLeon Romanovsky MODULE_DEVICE_TABLE(auxiliary, mlx5v_id_table); 315474c9729dSLeon Romanovsky 315574c9729dSLeon Romanovsky static struct auxiliary_driver mlx5v_driver = { 315674c9729dSLeon Romanovsky .name = "vnet", 315774c9729dSLeon Romanovsky .probe = mlx5v_probe, 315874c9729dSLeon Romanovsky .remove = mlx5v_remove, 315974c9729dSLeon Romanovsky .id_table = mlx5v_id_table, 316074c9729dSLeon Romanovsky }; 316174c9729dSLeon Romanovsky 316274c9729dSLeon Romanovsky module_auxiliary_driver(mlx5v_driver); 3163