11a86b377SEli Cohen // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 21a86b377SEli Cohen /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 31a86b377SEli Cohen 474c9729dSLeon Romanovsky #include <linux/module.h> 51a86b377SEli Cohen #include <linux/vdpa.h> 674c9729dSLeon Romanovsky #include <linux/vringh.h> 774c9729dSLeon Romanovsky #include <uapi/linux/virtio_net.h> 81a86b377SEli Cohen #include <uapi/linux/virtio_ids.h> 9a007d940SEli Cohen #include <uapi/linux/vdpa.h> 101a86b377SEli Cohen #include <linux/virtio_config.h> 1174c9729dSLeon Romanovsky #include <linux/auxiliary_bus.h> 1274c9729dSLeon Romanovsky #include <linux/mlx5/cq.h> 131a86b377SEli Cohen #include <linux/mlx5/qp.h> 141a86b377SEli Cohen #include <linux/mlx5/device.h> 1574c9729dSLeon Romanovsky #include <linux/mlx5/driver.h> 161a86b377SEli Cohen #include <linux/mlx5/vport.h> 171a86b377SEli Cohen #include <linux/mlx5/fs.h> 180aae392bSLeon Romanovsky #include <linux/mlx5/mlx5_ifc_vdpa.h> 197c9f131fSEli Cohen #include <linux/mlx5/mpfs.h> 201a86b377SEli Cohen #include "mlx5_vdpa.h" 211a86b377SEli Cohen 2274c9729dSLeon Romanovsky MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 2374c9729dSLeon Romanovsky MODULE_DESCRIPTION("Mellanox VDPA driver"); 2474c9729dSLeon Romanovsky MODULE_LICENSE("Dual BSD/GPL"); 2574c9729dSLeon Romanovsky 2674c9729dSLeon Romanovsky #define to_mlx5_vdpa_ndev(__mvdev) \ 2774c9729dSLeon Romanovsky container_of(__mvdev, struct mlx5_vdpa_net, mvdev) 281a86b377SEli Cohen #define to_mvdev(__vdev) container_of((__vdev), struct mlx5_vdpa_dev, vdev) 291a86b377SEli Cohen 301a86b377SEli Cohen #define VALID_FEATURES_MASK \ 31cbb52359SNathan Chancellor (BIT_ULL(VIRTIO_NET_F_CSUM) | BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | \ 32cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_MAC) | \ 33cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | \ 34cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | \ 35cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | BIT_ULL(VIRTIO_NET_F_HOST_ECN) | BIT_ULL(VIRTIO_NET_F_HOST_UFO) | \ 36cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | BIT_ULL(VIRTIO_NET_F_STATUS) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | \ 37cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_RX) | BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | \ 38cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE) | \ 39cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | \ 40cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_RSS) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY) | \ 41cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX) | BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY) | \ 42cbb52359SNathan Chancellor BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | \ 43cbb52359SNathan Chancellor BIT_ULL(VIRTIO_F_RING_PACKED) | BIT_ULL(VIRTIO_F_ORDER_PLATFORM) | BIT_ULL(VIRTIO_F_SR_IOV)) 441a86b377SEli Cohen 451a86b377SEli Cohen #define VALID_STATUS_MASK \ 461a86b377SEli Cohen (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \ 471a86b377SEli Cohen VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED) 481a86b377SEli Cohen 49e4fc6650SEli Cohen #define MLX5_FEATURE(_mvdev, _feature) (!!((_mvdev)->actual_features & BIT_ULL(_feature))) 50e4fc6650SEli Cohen 51baf2ad3fSEli Cohen #define MLX5V_UNTAGGED 0x1000 52baf2ad3fSEli Cohen 531a86b377SEli Cohen struct mlx5_vdpa_net_resources { 541a86b377SEli Cohen u32 tisn; 551a86b377SEli Cohen u32 tdn; 561a86b377SEli Cohen u32 tirn; 571a86b377SEli Cohen u32 rqtn; 581a86b377SEli Cohen bool valid; 591a86b377SEli Cohen }; 601a86b377SEli Cohen 611a86b377SEli Cohen struct mlx5_vdpa_cq_buf { 621a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 631a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 641a86b377SEli Cohen int cqe_size; 651a86b377SEli Cohen int nent; 661a86b377SEli Cohen }; 671a86b377SEli Cohen 681a86b377SEli Cohen struct mlx5_vdpa_cq { 691a86b377SEli Cohen struct mlx5_core_cq mcq; 701a86b377SEli Cohen struct mlx5_vdpa_cq_buf buf; 711a86b377SEli Cohen struct mlx5_db db; 721a86b377SEli Cohen int cqe; 731a86b377SEli Cohen }; 741a86b377SEli Cohen 751a86b377SEli Cohen struct mlx5_vdpa_umem { 761a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 771a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 781a86b377SEli Cohen int size; 791a86b377SEli Cohen u32 id; 801a86b377SEli Cohen }; 811a86b377SEli Cohen 821a86b377SEli Cohen struct mlx5_vdpa_qp { 831a86b377SEli Cohen struct mlx5_core_qp mqp; 841a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 851a86b377SEli Cohen struct mlx5_db db; 861a86b377SEli Cohen u16 head; 871a86b377SEli Cohen bool fw; 881a86b377SEli Cohen }; 891a86b377SEli Cohen 901a86b377SEli Cohen struct mlx5_vq_restore_info { 911a86b377SEli Cohen u32 num_ent; 921a86b377SEli Cohen u64 desc_addr; 931a86b377SEli Cohen u64 device_addr; 941a86b377SEli Cohen u64 driver_addr; 951a86b377SEli Cohen u16 avail_index; 96b35ccebeSEli Cohen u16 used_index; 971a86b377SEli Cohen bool ready; 981a86b377SEli Cohen bool restore; 991a86b377SEli Cohen }; 1001a86b377SEli Cohen 1011a86b377SEli Cohen struct mlx5_vdpa_virtqueue { 1021a86b377SEli Cohen bool ready; 1031a86b377SEli Cohen u64 desc_addr; 1041a86b377SEli Cohen u64 device_addr; 1051a86b377SEli Cohen u64 driver_addr; 1061a86b377SEli Cohen u32 num_ent; 1071a86b377SEli Cohen 1081a86b377SEli Cohen /* Resources for implementing the notification channel from the device 1091a86b377SEli Cohen * to the driver. fwqp is the firmware end of an RC connection; the 1102f72b226SXiang wangx * other end is vqqp used by the driver. cq is where completions are 1111a86b377SEli Cohen * reported. 1121a86b377SEli Cohen */ 1131a86b377SEli Cohen struct mlx5_vdpa_cq cq; 1141a86b377SEli Cohen struct mlx5_vdpa_qp fwqp; 1151a86b377SEli Cohen struct mlx5_vdpa_qp vqqp; 1161a86b377SEli Cohen 1171a86b377SEli Cohen /* umem resources are required for the virtqueue operation. They're use 1181a86b377SEli Cohen * is internal and they must be provided by the driver. 1191a86b377SEli Cohen */ 1201a86b377SEli Cohen struct mlx5_vdpa_umem umem1; 1211a86b377SEli Cohen struct mlx5_vdpa_umem umem2; 1221a86b377SEli Cohen struct mlx5_vdpa_umem umem3; 1231a86b377SEli Cohen 1241892a3d4SEli Cohen u32 counter_set_id; 1251a86b377SEli Cohen bool initialized; 1261a86b377SEli Cohen int index; 1271a86b377SEli Cohen u32 virtq_id; 1281a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 1291a86b377SEli Cohen u16 avail_idx; 130b35ccebeSEli Cohen u16 used_idx; 1311a86b377SEli Cohen int fw_state; 1321a86b377SEli Cohen 1331a86b377SEli Cohen /* keep last in the struct */ 1341a86b377SEli Cohen struct mlx5_vq_restore_info ri; 1351a86b377SEli Cohen }; 1361a86b377SEli Cohen 137e4fc6650SEli Cohen static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx) 138e4fc6650SEli Cohen { 139f8ae3a48SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) { 140f8ae3a48SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 141f8ae3a48SEli Cohen return idx < 2; 142f8ae3a48SEli Cohen else 143f8ae3a48SEli Cohen return idx < 3; 144f8ae3a48SEli Cohen } 145e4fc6650SEli Cohen 146f8ae3a48SEli Cohen return idx <= mvdev->max_idx; 147e4fc6650SEli Cohen } 148e4fc6650SEli Cohen 149baf2ad3fSEli Cohen #define MLX5V_MACVLAN_SIZE 256 150baf2ad3fSEli Cohen 1511a86b377SEli Cohen struct mlx5_vdpa_net { 1521a86b377SEli Cohen struct mlx5_vdpa_dev mvdev; 1531a86b377SEli Cohen struct mlx5_vdpa_net_resources res; 1541a86b377SEli Cohen struct virtio_net_config config; 15575560522SEli Cohen struct mlx5_vdpa_virtqueue *vqs; 15675560522SEli Cohen struct vdpa_callback *event_cbs; 1571a86b377SEli Cohen 1581a86b377SEli Cohen /* Serialize vq resources creation and destruction. This is required 1591a86b377SEli Cohen * since memory map might change and we need to destroy and create 1601a86b377SEli Cohen * resources while driver in operational. 1611a86b377SEli Cohen */ 162759ae7f9SEli Cohen struct rw_semaphore reslock; 1631a86b377SEli Cohen struct mlx5_flow_table *rxft; 1641a86b377SEli Cohen bool setup; 1655262912eSEli Cohen u32 cur_num_vqs; 166acde3929SEli Cohen u32 rqt_size; 167cae15c2eSEli Cohen bool nb_registered; 168edf747afSEli Cohen struct notifier_block nb; 169edf747afSEli Cohen struct vdpa_callback config_cb; 17055ebf0d6SJason Wang struct mlx5_vdpa_wq_ent cvq_ent; 171baf2ad3fSEli Cohen struct hlist_head macvlan_hash[MLX5V_MACVLAN_SIZE]; 172baf2ad3fSEli Cohen }; 173baf2ad3fSEli Cohen 174baf2ad3fSEli Cohen struct macvlan_node { 175baf2ad3fSEli Cohen struct hlist_node hlist; 176baf2ad3fSEli Cohen struct mlx5_flow_handle *ucast_rule; 177baf2ad3fSEli Cohen struct mlx5_flow_handle *mcast_rule; 178baf2ad3fSEli Cohen u64 macvlan; 1791a86b377SEli Cohen }; 1801a86b377SEli Cohen 1811a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev); 1821a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev); 183ae0428deSEli Cohen static int setup_driver(struct mlx5_vdpa_dev *mvdev); 1841a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev); 1851a86b377SEli Cohen 1861a86b377SEli Cohen static bool mlx5_vdpa_debug; 1871a86b377SEli Cohen 1885262912eSEli Cohen #define MLX5_CVQ_MAX_ENT 16 1895262912eSEli Cohen 1901a86b377SEli Cohen #define MLX5_LOG_VIO_FLAG(_feature) \ 1911a86b377SEli Cohen do { \ 192cbb52359SNathan Chancellor if (features & BIT_ULL(_feature)) \ 1931a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_feature); \ 1941a86b377SEli Cohen } while (0) 1951a86b377SEli Cohen 1961a86b377SEli Cohen #define MLX5_LOG_VIO_STAT(_status) \ 1971a86b377SEli Cohen do { \ 1981a86b377SEli Cohen if (status & (_status)) \ 1991a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_status); \ 2001a86b377SEli Cohen } while (0) 2011a86b377SEli Cohen 20252893733SEli Cohen /* TODO: cross-endian support */ 20352893733SEli Cohen static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev) 20452893733SEli Cohen { 20552893733SEli Cohen return virtio_legacy_is_little_endian() || 20652893733SEli Cohen (mvdev->actual_features & BIT_ULL(VIRTIO_F_VERSION_1)); 20752893733SEli Cohen } 20852893733SEli Cohen 20952893733SEli Cohen static u16 mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev *mvdev, __virtio16 val) 21052893733SEli Cohen { 21152893733SEli Cohen return __virtio16_to_cpu(mlx5_vdpa_is_little_endian(mvdev), val); 21252893733SEli Cohen } 21352893733SEli Cohen 21452893733SEli Cohen static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val) 21552893733SEli Cohen { 21652893733SEli Cohen return __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), val); 21752893733SEli Cohen } 21852893733SEli Cohen 2195262912eSEli Cohen static u16 ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev) 2205262912eSEli Cohen { 22152893733SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) 22252893733SEli Cohen return 2; 22352893733SEli Cohen 224acde3929SEli Cohen return mvdev->max_vqs; 2255262912eSEli Cohen } 2265262912eSEli Cohen 2275262912eSEli Cohen static bool is_ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev, u16 idx) 2285262912eSEli Cohen { 2295262912eSEli Cohen return idx == ctrl_vq_idx(mvdev); 2305262912eSEli Cohen } 2315262912eSEli Cohen 2321a86b377SEli Cohen static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set) 2331a86b377SEli Cohen { 2341a86b377SEli Cohen if (status & ~VALID_STATUS_MASK) 2351a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n", 2361a86b377SEli Cohen status & ~VALID_STATUS_MASK); 2371a86b377SEli Cohen 2381a86b377SEli Cohen if (!mlx5_vdpa_debug) 2391a86b377SEli Cohen return; 2401a86b377SEli Cohen 2411a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get"); 2421a86b377SEli Cohen if (set && !status) { 2431a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver resets the device\n"); 2441a86b377SEli Cohen return; 2451a86b377SEli Cohen } 2461a86b377SEli Cohen 2471a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE); 2481a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER); 2491a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK); 2501a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK); 2511a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET); 2521a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED); 2531a86b377SEli Cohen } 2541a86b377SEli Cohen 2551a86b377SEli Cohen static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set) 2561a86b377SEli Cohen { 2571a86b377SEli Cohen if (features & ~VALID_FEATURES_MASK) 2581a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n", 2591a86b377SEli Cohen features & ~VALID_FEATURES_MASK); 2601a86b377SEli Cohen 2611a86b377SEli Cohen if (!mlx5_vdpa_debug) 2621a86b377SEli Cohen return; 2631a86b377SEli Cohen 2641a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads"); 2651a86b377SEli Cohen if (!features) 2661a86b377SEli Cohen mlx5_vdpa_info(mvdev, "all feature bits are cleared\n"); 2671a86b377SEli Cohen 2681a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM); 2691a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM); 2701a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 2711a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU); 2721a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC); 2731a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4); 2741a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6); 2751a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN); 2761a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO); 2771a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4); 2781a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6); 2791a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN); 2801a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO); 2811a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF); 2821a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS); 2831a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ); 2841a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX); 2851a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN); 2861a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA); 2871a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE); 2881a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ); 2891a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR); 2901a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT); 2911a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS); 2921a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT); 2931a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY); 2941a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX); 2951a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY); 2961a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT); 2971a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1); 2981a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM); 2991a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED); 3001a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM); 3011a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV); 3021a86b377SEli Cohen } 3031a86b377SEli Cohen 3041a86b377SEli Cohen static int create_tis(struct mlx5_vdpa_net *ndev) 3051a86b377SEli Cohen { 3061a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; 3071a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 3081a86b377SEli Cohen void *tisc; 3091a86b377SEli Cohen int err; 3101a86b377SEli Cohen 3111a86b377SEli Cohen tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3121a86b377SEli Cohen MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); 3131a86b377SEli Cohen err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); 3141a86b377SEli Cohen if (err) 3151a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err); 3161a86b377SEli Cohen 3171a86b377SEli Cohen return err; 3181a86b377SEli Cohen } 3191a86b377SEli Cohen 3201a86b377SEli Cohen static void destroy_tis(struct mlx5_vdpa_net *ndev) 3211a86b377SEli Cohen { 3221a86b377SEli Cohen mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); 3231a86b377SEli Cohen } 3241a86b377SEli Cohen 3251a86b377SEli Cohen #define MLX5_VDPA_CQE_SIZE 64 3261a86b377SEli Cohen #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE) 3271a86b377SEli Cohen 3281a86b377SEli Cohen static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) 3291a86b377SEli Cohen { 3301a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &buf->frag_buf; 3311a86b377SEli Cohen u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE; 3321a86b377SEli Cohen u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE; 3331a86b377SEli Cohen int err; 3341a86b377SEli Cohen 3351a86b377SEli Cohen err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, 3361a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3371a86b377SEli Cohen if (err) 3381a86b377SEli Cohen return err; 3391a86b377SEli Cohen 3401a86b377SEli Cohen mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc); 3411a86b377SEli Cohen 3421a86b377SEli Cohen buf->cqe_size = MLX5_VDPA_CQE_SIZE; 3431a86b377SEli Cohen buf->nent = nent; 3441a86b377SEli Cohen 3451a86b377SEli Cohen return 0; 3461a86b377SEli Cohen } 3471a86b377SEli Cohen 3481a86b377SEli Cohen static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) 3491a86b377SEli Cohen { 3501a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &umem->frag_buf; 3511a86b377SEli Cohen 3521a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, 3531a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3541a86b377SEli Cohen } 3551a86b377SEli Cohen 3561a86b377SEli Cohen static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) 3571a86b377SEli Cohen { 3581a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); 3591a86b377SEli Cohen } 3601a86b377SEli Cohen 3611a86b377SEli Cohen static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n) 3621a86b377SEli Cohen { 3631a86b377SEli Cohen return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n); 3641a86b377SEli Cohen } 3651a86b377SEli Cohen 3661a86b377SEli Cohen static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf) 3671a86b377SEli Cohen { 3681a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 3691a86b377SEli Cohen void *cqe; 3701a86b377SEli Cohen int i; 3711a86b377SEli Cohen 3721a86b377SEli Cohen for (i = 0; i < buf->nent; i++) { 3731a86b377SEli Cohen cqe = get_cqe(vcq, i); 3741a86b377SEli Cohen cqe64 = cqe; 3751a86b377SEli Cohen cqe64->op_own = MLX5_CQE_INVALID << 4; 3761a86b377SEli Cohen } 3771a86b377SEli Cohen } 3781a86b377SEli Cohen 3791a86b377SEli Cohen static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n) 3801a86b377SEli Cohen { 3811a86b377SEli Cohen struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1)); 3821a86b377SEli Cohen 3831a86b377SEli Cohen if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && 3841a86b377SEli Cohen !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe))) 3851a86b377SEli Cohen return cqe64; 3861a86b377SEli Cohen 3871a86b377SEli Cohen return NULL; 3881a86b377SEli Cohen } 3891a86b377SEli Cohen 3901a86b377SEli Cohen static void rx_post(struct mlx5_vdpa_qp *vqp, int n) 3911a86b377SEli Cohen { 3921a86b377SEli Cohen vqp->head += n; 3931a86b377SEli Cohen vqp->db.db[0] = cpu_to_be32(vqp->head); 3941a86b377SEli Cohen } 3951a86b377SEli Cohen 3961a86b377SEli Cohen static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, 3971a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq, u32 num_ent) 3981a86b377SEli Cohen { 3991a86b377SEli Cohen struct mlx5_vdpa_qp *vqp; 4001a86b377SEli Cohen __be64 *pas; 4011a86b377SEli Cohen void *qpc; 4021a86b377SEli Cohen 4031a86b377SEli Cohen vqp = fw ? &mvq->fwqp : &mvq->vqqp; 4041a86b377SEli Cohen MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); 4051a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4061a86b377SEli Cohen if (vqp->fw) { 4071a86b377SEli Cohen /* Firmware QP is allocated by the driver for the firmware's 4081a86b377SEli Cohen * use so we can skip part of the params as they will be chosen by firmware 4091a86b377SEli Cohen */ 4101a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4111a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ); 4121a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 4131a86b377SEli Cohen return; 4141a86b377SEli Cohen } 4151a86b377SEli Cohen 4161a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4171a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4181a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4191a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4201a86b377SEli Cohen MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); 4211a86b377SEli Cohen MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 4221a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 4231a86b377SEli Cohen MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn); 4241a86b377SEli Cohen MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent)); 4251a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 4261a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas); 4271a86b377SEli Cohen mlx5_fill_page_frag_array(&vqp->frag_buf, pas); 4281a86b377SEli Cohen } 4291a86b377SEli Cohen 4301a86b377SEli Cohen static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) 4311a86b377SEli Cohen { 4321a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, 4331a86b377SEli Cohen num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf, 4341a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 4351a86b377SEli Cohen } 4361a86b377SEli Cohen 4371a86b377SEli Cohen static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 4381a86b377SEli Cohen { 4391a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); 4401a86b377SEli Cohen } 4411a86b377SEli Cohen 4421a86b377SEli Cohen static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 4431a86b377SEli Cohen struct mlx5_vdpa_qp *vqp) 4441a86b377SEli Cohen { 4451a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 4461a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_qp_in); 4471a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 4481a86b377SEli Cohen void *qpc; 4491a86b377SEli Cohen void *in; 4501a86b377SEli Cohen int err; 4511a86b377SEli Cohen 4521a86b377SEli Cohen if (!vqp->fw) { 4531a86b377SEli Cohen vqp = &mvq->vqqp; 4541a86b377SEli Cohen err = rq_buf_alloc(ndev, vqp, mvq->num_ent); 4551a86b377SEli Cohen if (err) 4561a86b377SEli Cohen return err; 4571a86b377SEli Cohen 4581a86b377SEli Cohen err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); 4591a86b377SEli Cohen if (err) 4601a86b377SEli Cohen goto err_db; 4611a86b377SEli Cohen inlen += vqp->frag_buf.npages * sizeof(__be64); 4621a86b377SEli Cohen } 4631a86b377SEli Cohen 4641a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 4651a86b377SEli Cohen if (!in) { 4661a86b377SEli Cohen err = -ENOMEM; 4671a86b377SEli Cohen goto err_kzalloc; 4681a86b377SEli Cohen } 4691a86b377SEli Cohen 4701a86b377SEli Cohen qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); 4711a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4721a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4731a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4741a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4751a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4761a86b377SEli Cohen if (!vqp->fw) 4771a86b377SEli Cohen MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma); 4781a86b377SEli Cohen MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 4791a86b377SEli Cohen err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 4801a86b377SEli Cohen kfree(in); 4811a86b377SEli Cohen if (err) 4821a86b377SEli Cohen goto err_kzalloc; 4831a86b377SEli Cohen 4841a86b377SEli Cohen vqp->mqp.uid = ndev->mvdev.res.uid; 4851a86b377SEli Cohen vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn); 4861a86b377SEli Cohen 4871a86b377SEli Cohen if (!vqp->fw) 4881a86b377SEli Cohen rx_post(vqp, mvq->num_ent); 4891a86b377SEli Cohen 4901a86b377SEli Cohen return 0; 4911a86b377SEli Cohen 4921a86b377SEli Cohen err_kzalloc: 4931a86b377SEli Cohen if (!vqp->fw) 4941a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 4951a86b377SEli Cohen err_db: 4961a86b377SEli Cohen if (!vqp->fw) 4971a86b377SEli Cohen rq_buf_free(ndev, vqp); 4981a86b377SEli Cohen 4991a86b377SEli Cohen return err; 5001a86b377SEli Cohen } 5011a86b377SEli Cohen 5021a86b377SEli Cohen static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 5031a86b377SEli Cohen { 5041a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 5051a86b377SEli Cohen 5061a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 5071a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn); 5081a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); 5091a86b377SEli Cohen if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) 5101a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); 5111a86b377SEli Cohen if (!vqp->fw) { 5121a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 5131a86b377SEli Cohen rq_buf_free(ndev, vqp); 5141a86b377SEli Cohen } 5151a86b377SEli Cohen } 5161a86b377SEli Cohen 5171a86b377SEli Cohen static void *next_cqe_sw(struct mlx5_vdpa_cq *cq) 5181a86b377SEli Cohen { 5191a86b377SEli Cohen return get_sw_cqe(cq, cq->mcq.cons_index); 5201a86b377SEli Cohen } 5211a86b377SEli Cohen 5221a86b377SEli Cohen static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq) 5231a86b377SEli Cohen { 5241a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 5251a86b377SEli Cohen 5261a86b377SEli Cohen cqe64 = next_cqe_sw(vcq); 5271a86b377SEli Cohen if (!cqe64) 5281a86b377SEli Cohen return -EAGAIN; 5291a86b377SEli Cohen 5301a86b377SEli Cohen vcq->mcq.cons_index++; 5311a86b377SEli Cohen return 0; 5321a86b377SEli Cohen } 5331a86b377SEli Cohen 5341a86b377SEli Cohen static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num) 5351a86b377SEli Cohen { 536db296d25SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 537db296d25SEli Cohen struct vdpa_callback *event_cb; 538db296d25SEli Cohen 539db296d25SEli Cohen event_cb = &ndev->event_cbs[mvq->index]; 5401a86b377SEli Cohen mlx5_cq_set_ci(&mvq->cq.mcq); 54183ef73b2SEli Cohen 54283ef73b2SEli Cohen /* make sure CQ cosumer update is visible to the hardware before updating 54383ef73b2SEli Cohen * RX doorbell record. 54483ef73b2SEli Cohen */ 54583ef73b2SEli Cohen dma_wmb(); 5461a86b377SEli Cohen rx_post(&mvq->vqqp, num); 547db296d25SEli Cohen if (event_cb->callback) 548db296d25SEli Cohen event_cb->callback(event_cb->private); 5491a86b377SEli Cohen } 5501a86b377SEli Cohen 5511a86b377SEli Cohen static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) 5521a86b377SEli Cohen { 5531a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq); 5541a86b377SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 5551a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5561a86b377SEli Cohen int num = 0; 5571a86b377SEli Cohen 5581a86b377SEli Cohen while (!mlx5_vdpa_poll_one(&mvq->cq)) { 5591a86b377SEli Cohen num++; 5601a86b377SEli Cohen if (num > mvq->num_ent / 2) { 5611a86b377SEli Cohen /* If completions keep coming while we poll, we want to 5621a86b377SEli Cohen * let the hardware know that we consumed them by 5631a86b377SEli Cohen * updating the doorbell record. We also let vdpa core 5641a86b377SEli Cohen * know about this so it passes it on the virtio driver 5651a86b377SEli Cohen * on the guest. 5661a86b377SEli Cohen */ 5671a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 5681a86b377SEli Cohen num = 0; 5691a86b377SEli Cohen } 5701a86b377SEli Cohen } 5711a86b377SEli Cohen 5721a86b377SEli Cohen if (num) 5731a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 5741a86b377SEli Cohen 5751a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 5761a86b377SEli Cohen } 5771a86b377SEli Cohen 5781a86b377SEli Cohen static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) 5791a86b377SEli Cohen { 5801a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 5811a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 5821a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5831a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 5841a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 5851a86b377SEli Cohen __be64 *pas; 5861a86b377SEli Cohen int inlen; 5871a86b377SEli Cohen void *cqc; 5881a86b377SEli Cohen void *in; 5891a86b377SEli Cohen int err; 5901a86b377SEli Cohen int eqn; 5911a86b377SEli Cohen 5921a86b377SEli Cohen err = mlx5_db_alloc(mdev, &vcq->db); 5931a86b377SEli Cohen if (err) 5941a86b377SEli Cohen return err; 5951a86b377SEli Cohen 5961a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 5971a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 5981a86b377SEli Cohen vcq->mcq.cqe_sz = 64; 5991a86b377SEli Cohen 6001a86b377SEli Cohen err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); 6011a86b377SEli Cohen if (err) 6021a86b377SEli Cohen goto err_db; 6031a86b377SEli Cohen 6041a86b377SEli Cohen cq_frag_buf_init(vcq, &vcq->buf); 6051a86b377SEli Cohen 6061a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 6071a86b377SEli Cohen MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages; 6081a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 6091a86b377SEli Cohen if (!in) { 6101a86b377SEli Cohen err = -ENOMEM; 6111a86b377SEli Cohen goto err_vzalloc; 6121a86b377SEli Cohen } 6131a86b377SEli Cohen 6141a86b377SEli Cohen MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); 6151a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 6161a86b377SEli Cohen mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas); 6171a86b377SEli Cohen 6181a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 6191a86b377SEli Cohen MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 6201a86b377SEli Cohen 6211a86b377SEli Cohen /* Use vector 0 by default. Consider adding code to choose least used 6221a86b377SEli Cohen * vector. 6231a86b377SEli Cohen */ 624563476aeSShay Drory err = mlx5_vector2eqn(mdev, 0, &eqn); 6251a86b377SEli Cohen if (err) 6261a86b377SEli Cohen goto err_vec; 6271a86b377SEli Cohen 6281a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 6291a86b377SEli Cohen MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent)); 6301a86b377SEli Cohen MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); 631616d5769STal Gilboa MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 6321a86b377SEli Cohen MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma); 6331a86b377SEli Cohen 6341a86b377SEli Cohen err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out)); 6351a86b377SEli Cohen if (err) 6361a86b377SEli Cohen goto err_vec; 6371a86b377SEli Cohen 6381a86b377SEli Cohen vcq->mcq.comp = mlx5_vdpa_cq_comp; 6391a86b377SEli Cohen vcq->cqe = num_ent; 6401a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 6411a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 6421a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 6431a86b377SEli Cohen kfree(in); 6441a86b377SEli Cohen return 0; 6451a86b377SEli Cohen 6461a86b377SEli Cohen err_vec: 6471a86b377SEli Cohen kfree(in); 6481a86b377SEli Cohen err_vzalloc: 6491a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 6501a86b377SEli Cohen err_db: 6511a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 6521a86b377SEli Cohen return err; 6531a86b377SEli Cohen } 6541a86b377SEli Cohen 6551a86b377SEli Cohen static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) 6561a86b377SEli Cohen { 6571a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 6581a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 6591a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 6601a86b377SEli Cohen 6611a86b377SEli Cohen if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) { 6621a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); 6631a86b377SEli Cohen return; 6641a86b377SEli Cohen } 6651a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 6661a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 6671a86b377SEli Cohen } 6681a86b377SEli Cohen 66971ab6a7cSEli Cohen static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, 6701a86b377SEli Cohen struct mlx5_vdpa_umem **umemp) 6711a86b377SEli Cohen { 6721a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 6731a86b377SEli Cohen int p_a; 6741a86b377SEli Cohen int p_b; 6751a86b377SEli Cohen 6761a86b377SEli Cohen switch (num) { 6771a86b377SEli Cohen case 1: 6781a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a); 6791a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b); 6801a86b377SEli Cohen *umemp = &mvq->umem1; 6811a86b377SEli Cohen break; 6821a86b377SEli Cohen case 2: 6831a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a); 6841a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b); 6851a86b377SEli Cohen *umemp = &mvq->umem2; 6861a86b377SEli Cohen break; 6871a86b377SEli Cohen case 3: 6881a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a); 6891a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b); 6901a86b377SEli Cohen *umemp = &mvq->umem3; 6911a86b377SEli Cohen break; 6921a86b377SEli Cohen } 69371ab6a7cSEli Cohen (*umemp)->size = p_a * mvq->num_ent + p_b; 6941a86b377SEli Cohen } 6951a86b377SEli Cohen 6961a86b377SEli Cohen static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) 6971a86b377SEli Cohen { 6981a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); 6991a86b377SEli Cohen } 7001a86b377SEli Cohen 7011a86b377SEli Cohen static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 7021a86b377SEli Cohen { 7031a86b377SEli Cohen int inlen; 7041a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {}; 7051a86b377SEli Cohen void *um; 7061a86b377SEli Cohen void *in; 7071a86b377SEli Cohen int err; 7081a86b377SEli Cohen __be64 *pas; 7091a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 7101a86b377SEli Cohen 71171ab6a7cSEli Cohen set_umem_size(ndev, mvq, num, &umem); 71271ab6a7cSEli Cohen err = umem_frag_buf_alloc(ndev, umem, umem->size); 7131a86b377SEli Cohen if (err) 7141a86b377SEli Cohen return err; 7151a86b377SEli Cohen 7161a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages; 7171a86b377SEli Cohen 7181a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 7191a86b377SEli Cohen if (!in) { 7201a86b377SEli Cohen err = -ENOMEM; 7211a86b377SEli Cohen goto err_in; 7221a86b377SEli Cohen } 7231a86b377SEli Cohen 7241a86b377SEli Cohen MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM); 7251a86b377SEli Cohen MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); 7261a86b377SEli Cohen um = MLX5_ADDR_OF(create_umem_in, in, umem); 7271a86b377SEli Cohen MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 7281a86b377SEli Cohen MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages); 7291a86b377SEli Cohen 7301a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]); 7311a86b377SEli Cohen mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW); 7321a86b377SEli Cohen 7331a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 7341a86b377SEli Cohen if (err) { 7351a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); 7361a86b377SEli Cohen goto err_cmd; 7371a86b377SEli Cohen } 7381a86b377SEli Cohen 7391a86b377SEli Cohen kfree(in); 7401a86b377SEli Cohen umem->id = MLX5_GET(create_umem_out, out, umem_id); 7411a86b377SEli Cohen 7421a86b377SEli Cohen return 0; 7431a86b377SEli Cohen 7441a86b377SEli Cohen err_cmd: 7451a86b377SEli Cohen kfree(in); 7461a86b377SEli Cohen err_in: 7471a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7481a86b377SEli Cohen return err; 7491a86b377SEli Cohen } 7501a86b377SEli Cohen 7511a86b377SEli Cohen static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 7521a86b377SEli Cohen { 7531a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {}; 7541a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {}; 7551a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 7561a86b377SEli Cohen 7571a86b377SEli Cohen switch (num) { 7581a86b377SEli Cohen case 1: 7591a86b377SEli Cohen umem = &mvq->umem1; 7601a86b377SEli Cohen break; 7611a86b377SEli Cohen case 2: 7621a86b377SEli Cohen umem = &mvq->umem2; 7631a86b377SEli Cohen break; 7641a86b377SEli Cohen case 3: 7651a86b377SEli Cohen umem = &mvq->umem3; 7661a86b377SEli Cohen break; 7671a86b377SEli Cohen } 7681a86b377SEli Cohen 7691a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM); 7701a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, umem_id, umem->id); 7711a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 7721a86b377SEli Cohen return; 7731a86b377SEli Cohen 7741a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7751a86b377SEli Cohen } 7761a86b377SEli Cohen 7771a86b377SEli Cohen static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7781a86b377SEli Cohen { 7791a86b377SEli Cohen int num; 7801a86b377SEli Cohen int err; 7811a86b377SEli Cohen 7821a86b377SEli Cohen for (num = 1; num <= 3; num++) { 7831a86b377SEli Cohen err = create_umem(ndev, mvq, num); 7841a86b377SEli Cohen if (err) 7851a86b377SEli Cohen goto err_umem; 7861a86b377SEli Cohen } 7871a86b377SEli Cohen return 0; 7881a86b377SEli Cohen 7891a86b377SEli Cohen err_umem: 7901a86b377SEli Cohen for (num--; num > 0; num--) 7911a86b377SEli Cohen umem_destroy(ndev, mvq, num); 7921a86b377SEli Cohen 7931a86b377SEli Cohen return err; 7941a86b377SEli Cohen } 7951a86b377SEli Cohen 7961a86b377SEli Cohen static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7971a86b377SEli Cohen { 7981a86b377SEli Cohen int num; 7991a86b377SEli Cohen 8001a86b377SEli Cohen for (num = 3; num > 0; num--) 8011a86b377SEli Cohen umem_destroy(ndev, mvq, num); 8021a86b377SEli Cohen } 8031a86b377SEli Cohen 8041a86b377SEli Cohen static int get_queue_type(struct mlx5_vdpa_net *ndev) 8051a86b377SEli Cohen { 8061a86b377SEli Cohen u32 type_mask; 8071a86b377SEli Cohen 8081a86b377SEli Cohen type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); 8091a86b377SEli Cohen 8101a86b377SEli Cohen /* prefer split queue */ 811879753c8SEli Cohen if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT) 8121a86b377SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT; 813879753c8SEli Cohen 814879753c8SEli Cohen WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED)); 815879753c8SEli Cohen 816879753c8SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED; 8171a86b377SEli Cohen } 8181a86b377SEli Cohen 8191a86b377SEli Cohen static bool vq_is_tx(u16 idx) 8201a86b377SEli Cohen { 8211a86b377SEli Cohen return idx % 2; 8221a86b377SEli Cohen } 8231a86b377SEli Cohen 8241a86b377SEli Cohen static u16 get_features_12_3(u64 features) 8251a86b377SEli Cohen { 826cbb52359SNathan Chancellor return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) | 827cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) | 828cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) | 829cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6); 8301a86b377SEli Cohen } 8311a86b377SEli Cohen 8321892a3d4SEli Cohen static bool counters_supported(const struct mlx5_vdpa_dev *mvdev) 8331892a3d4SEli Cohen { 8341892a3d4SEli Cohen return MLX5_CAP_GEN_64(mvdev->mdev, general_obj_types) & 8351892a3d4SEli Cohen BIT_ULL(MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 8361892a3d4SEli Cohen } 8371892a3d4SEli Cohen 8381a86b377SEli Cohen static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 8391a86b377SEli Cohen { 8401a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); 8411a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {}; 8421a86b377SEli Cohen void *obj_context; 8431a86b377SEli Cohen void *cmd_hdr; 8441a86b377SEli Cohen void *vq_ctx; 8451a86b377SEli Cohen void *in; 8461a86b377SEli Cohen int err; 8471a86b377SEli Cohen 8481a86b377SEli Cohen err = umems_create(ndev, mvq); 8491a86b377SEli Cohen if (err) 8501a86b377SEli Cohen return err; 8511a86b377SEli Cohen 8521a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 8531a86b377SEli Cohen if (!in) { 8541a86b377SEli Cohen err = -ENOMEM; 8551a86b377SEli Cohen goto err_alloc; 8561a86b377SEli Cohen } 8571a86b377SEli Cohen 8581a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr); 8591a86b377SEli Cohen 8601a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 8611a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 8621a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 8631a86b377SEli Cohen 8641a86b377SEli Cohen obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context); 8651a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); 866b35ccebeSEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); 8671a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, 8681a86b377SEli Cohen get_features_12_3(ndev->mvdev.actual_features)); 8691a86b377SEli Cohen vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context); 8701a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); 8711a86b377SEli Cohen 8721a86b377SEli Cohen if (vq_is_tx(mvq->index)) 8731a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); 8741a86b377SEli Cohen 8751a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 8761a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); 8771a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 8781a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); 8791a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 8804b454a82SEli Cohen !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); 8811a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); 8821a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); 8831a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); 88483fec3f1SAharon Landau MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey); 8851a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); 8861a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); 8871a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id); 888e3011776SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem2.size); 8891a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id); 890e3011776SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem3.size); 8911a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); 8921892a3d4SEli Cohen if (counters_supported(&ndev->mvdev)) 8931892a3d4SEli Cohen MLX5_SET(virtio_q, vq_ctx, counter_set_id, mvq->counter_set_id); 8941a86b377SEli Cohen 8951a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 8961a86b377SEli Cohen if (err) 8971a86b377SEli Cohen goto err_cmd; 8981a86b377SEli Cohen 899cae15c2eSEli Cohen mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT; 9001a86b377SEli Cohen kfree(in); 9011a86b377SEli Cohen mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 9021a86b377SEli Cohen 9031a86b377SEli Cohen return 0; 9041a86b377SEli Cohen 9051a86b377SEli Cohen err_cmd: 9061a86b377SEli Cohen kfree(in); 9071a86b377SEli Cohen err_alloc: 9081a86b377SEli Cohen umems_destroy(ndev, mvq); 9091a86b377SEli Cohen return err; 9101a86b377SEli Cohen } 9111a86b377SEli Cohen 9121a86b377SEli Cohen static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 9131a86b377SEli Cohen { 9141a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {}; 9151a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {}; 9161a86b377SEli Cohen 9171a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode, 9181a86b377SEli Cohen MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 9191a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id); 9201a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); 9211a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type, 9221a86b377SEli Cohen MLX5_OBJ_TYPE_VIRTIO_NET_Q); 9231a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { 9241a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); 9251a86b377SEli Cohen return; 9261a86b377SEli Cohen } 927cae15c2eSEli Cohen mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; 9281a86b377SEli Cohen umems_destroy(ndev, mvq); 9291a86b377SEli Cohen } 9301a86b377SEli Cohen 9311a86b377SEli Cohen static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 9321a86b377SEli Cohen { 9331a86b377SEli Cohen return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn; 9341a86b377SEli Cohen } 9351a86b377SEli Cohen 9361a86b377SEli Cohen static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 9371a86b377SEli Cohen { 9381a86b377SEli Cohen return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn; 9391a86b377SEli Cohen } 9401a86b377SEli Cohen 9411a86b377SEli Cohen static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, 9421a86b377SEli Cohen int *outlen, u32 qpn, u32 rqpn) 9431a86b377SEli Cohen { 9441a86b377SEli Cohen void *qpc; 9451a86b377SEli Cohen void *pp; 9461a86b377SEli Cohen 9471a86b377SEli Cohen switch (cmd) { 9481a86b377SEli Cohen case MLX5_CMD_OP_2RST_QP: 9491a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in); 9501a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out); 9511a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9521a86b377SEli Cohen *out = kzalloc(*outlen, GFP_KERNEL); 953f31231bfSColin Ian King if (!*in || !*out) 9541a86b377SEli Cohen goto outerr; 9551a86b377SEli Cohen 9561a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, opcode, cmd); 9571a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); 9581a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, qpn, qpn); 9591a86b377SEli Cohen break; 9601a86b377SEli Cohen case MLX5_CMD_OP_RST2INIT_QP: 9611a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in); 9621a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out); 9631a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9641a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL); 965f31231bfSColin Ian King if (!*in || !*out) 9661a86b377SEli Cohen goto outerr; 9671a86b377SEli Cohen 9681a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, opcode, cmd); 9691a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); 9701a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, qpn, qpn); 9711a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9721a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9731a86b377SEli Cohen MLX5_SET(qpc, qpc, rwe, 1); 9741a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9751a86b377SEli Cohen MLX5_SET(ads, pp, vhca_port_num, 1); 9761a86b377SEli Cohen break; 9771a86b377SEli Cohen case MLX5_CMD_OP_INIT2RTR_QP: 9781a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in); 9791a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out); 9801a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9811a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL); 982f31231bfSColin Ian King if (!*in || !*out) 9831a86b377SEli Cohen goto outerr; 9841a86b377SEli Cohen 9851a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, opcode, cmd); 9861a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); 9871a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, qpn, qpn); 9881a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9891a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 9901a86b377SEli Cohen MLX5_SET(qpc, qpc, log_msg_max, 30); 9911a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9921a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9931a86b377SEli Cohen MLX5_SET(ads, pp, fl, 1); 9941a86b377SEli Cohen break; 9951a86b377SEli Cohen case MLX5_CMD_OP_RTR2RTS_QP: 9961a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in); 9971a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out); 9981a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9991a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL); 1000f31231bfSColin Ian King if (!*in || !*out) 10011a86b377SEli Cohen goto outerr; 10021a86b377SEli Cohen 10031a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd); 10041a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); 10051a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn); 10061a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 10071a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 10081a86b377SEli Cohen MLX5_SET(ads, pp, ack_timeout, 14); 10091a86b377SEli Cohen MLX5_SET(qpc, qpc, retry_count, 7); 10101a86b377SEli Cohen MLX5_SET(qpc, qpc, rnr_retry, 7); 10111a86b377SEli Cohen break; 10121a86b377SEli Cohen default: 1013f31231bfSColin Ian King goto outerr_nullify; 10141a86b377SEli Cohen } 10151a86b377SEli Cohen 10161a86b377SEli Cohen return; 10171a86b377SEli Cohen 10181a86b377SEli Cohen outerr: 10191a86b377SEli Cohen kfree(*in); 10201a86b377SEli Cohen kfree(*out); 1021f31231bfSColin Ian King outerr_nullify: 10221a86b377SEli Cohen *in = NULL; 10231a86b377SEli Cohen *out = NULL; 10241a86b377SEli Cohen } 10251a86b377SEli Cohen 10261a86b377SEli Cohen static void free_inout(void *in, void *out) 10271a86b377SEli Cohen { 10281a86b377SEli Cohen kfree(in); 10291a86b377SEli Cohen kfree(out); 10301a86b377SEli Cohen } 10311a86b377SEli Cohen 10321a86b377SEli Cohen /* Two QPs are used by each virtqueue. One is used by the driver and one by 10331a86b377SEli Cohen * firmware. The fw argument indicates whether the subjected QP is the one used 10341a86b377SEli Cohen * by firmware. 10351a86b377SEli Cohen */ 10361a86b377SEli Cohen static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) 10371a86b377SEli Cohen { 10381a86b377SEli Cohen int outlen; 10391a86b377SEli Cohen int inlen; 10401a86b377SEli Cohen void *out; 10411a86b377SEli Cohen void *in; 10421a86b377SEli Cohen int err; 10431a86b377SEli Cohen 10441a86b377SEli Cohen alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); 10451a86b377SEli Cohen if (!in || !out) 10461a86b377SEli Cohen return -ENOMEM; 10471a86b377SEli Cohen 10481a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); 10491a86b377SEli Cohen free_inout(in, out); 10501a86b377SEli Cohen return err; 10511a86b377SEli Cohen } 10521a86b377SEli Cohen 10531a86b377SEli Cohen static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 10541a86b377SEli Cohen { 10551a86b377SEli Cohen int err; 10561a86b377SEli Cohen 10571a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); 10581a86b377SEli Cohen if (err) 10591a86b377SEli Cohen return err; 10601a86b377SEli Cohen 10611a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); 10621a86b377SEli Cohen if (err) 10631a86b377SEli Cohen return err; 10641a86b377SEli Cohen 10651a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); 10661a86b377SEli Cohen if (err) 10671a86b377SEli Cohen return err; 10681a86b377SEli Cohen 10691a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); 10701a86b377SEli Cohen if (err) 10711a86b377SEli Cohen return err; 10721a86b377SEli Cohen 10731a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); 10741a86b377SEli Cohen if (err) 10751a86b377SEli Cohen return err; 10761a86b377SEli Cohen 10771a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); 10781a86b377SEli Cohen if (err) 10791a86b377SEli Cohen return err; 10801a86b377SEli Cohen 10811a86b377SEli Cohen return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); 10821a86b377SEli Cohen } 10831a86b377SEli Cohen 10841a86b377SEli Cohen struct mlx5_virtq_attr { 10851a86b377SEli Cohen u8 state; 10861a86b377SEli Cohen u16 available_index; 1087b35ccebeSEli Cohen u16 used_index; 10881a86b377SEli Cohen }; 10891a86b377SEli Cohen 10901a86b377SEli Cohen static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 10911a86b377SEli Cohen struct mlx5_virtq_attr *attr) 10921a86b377SEli Cohen { 10931a86b377SEli Cohen int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out); 10941a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {}; 10951a86b377SEli Cohen void *out; 10961a86b377SEli Cohen void *obj_context; 10971a86b377SEli Cohen void *cmd_hdr; 10981a86b377SEli Cohen int err; 10991a86b377SEli Cohen 11001a86b377SEli Cohen out = kzalloc(outlen, GFP_KERNEL); 11011a86b377SEli Cohen if (!out) 11021a86b377SEli Cohen return -ENOMEM; 11031a86b377SEli Cohen 11041a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr); 11051a86b377SEli Cohen 11061a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 11071a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 11081a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 11091a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 11101a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen); 11111a86b377SEli Cohen if (err) 11121a86b377SEli Cohen goto err_cmd; 11131a86b377SEli Cohen 11141a86b377SEli Cohen obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context); 11151a86b377SEli Cohen memset(attr, 0, sizeof(*attr)); 11161a86b377SEli Cohen attr->state = MLX5_GET(virtio_net_q_object, obj_context, state); 11171a86b377SEli Cohen attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index); 1118b35ccebeSEli Cohen attr->used_index = MLX5_GET(virtio_net_q_object, obj_context, hw_used_index); 11191a86b377SEli Cohen kfree(out); 11201a86b377SEli Cohen return 0; 11211a86b377SEli Cohen 11221a86b377SEli Cohen err_cmd: 11231a86b377SEli Cohen kfree(out); 11241a86b377SEli Cohen return err; 11251a86b377SEli Cohen } 11261a86b377SEli Cohen 1127cae15c2eSEli Cohen static bool is_valid_state_change(int oldstate, int newstate) 1128cae15c2eSEli Cohen { 1129cae15c2eSEli Cohen switch (oldstate) { 1130cae15c2eSEli Cohen case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT: 1131cae15c2eSEli Cohen return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY; 1132cae15c2eSEli Cohen case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY: 1133cae15c2eSEli Cohen return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND; 1134cae15c2eSEli Cohen case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND: 1135cae15c2eSEli Cohen case MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR: 1136cae15c2eSEli Cohen default: 1137cae15c2eSEli Cohen return false; 1138cae15c2eSEli Cohen } 1139cae15c2eSEli Cohen } 1140cae15c2eSEli Cohen 11411a86b377SEli Cohen static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) 11421a86b377SEli Cohen { 11431a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); 11441a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {}; 11451a86b377SEli Cohen void *obj_context; 11461a86b377SEli Cohen void *cmd_hdr; 11471a86b377SEli Cohen void *in; 11481a86b377SEli Cohen int err; 11491a86b377SEli Cohen 1150cae15c2eSEli Cohen if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_NONE) 1151cae15c2eSEli Cohen return 0; 1152cae15c2eSEli Cohen 1153cae15c2eSEli Cohen if (!is_valid_state_change(mvq->fw_state, state)) 1154cae15c2eSEli Cohen return -EINVAL; 1155cae15c2eSEli Cohen 11561a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 11571a86b377SEli Cohen if (!in) 11581a86b377SEli Cohen return -ENOMEM; 11591a86b377SEli Cohen 11601a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr); 11611a86b377SEli Cohen 11621a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); 11631a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 11641a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 11651a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 11661a86b377SEli Cohen 11671a86b377SEli Cohen obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context); 11681a86b377SEli Cohen MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select, 11691a86b377SEli Cohen MLX5_VIRTQ_MODIFY_MASK_STATE); 11701a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, state, state); 11711a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 11721a86b377SEli Cohen kfree(in); 11731a86b377SEli Cohen if (!err) 11741a86b377SEli Cohen mvq->fw_state = state; 11751a86b377SEli Cohen 11761a86b377SEli Cohen return err; 11771a86b377SEli Cohen } 11781a86b377SEli Cohen 11791892a3d4SEli Cohen static int counter_set_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11801892a3d4SEli Cohen { 11811892a3d4SEli Cohen u32 in[MLX5_ST_SZ_DW(create_virtio_q_counters_in)] = {}; 11821892a3d4SEli Cohen u32 out[MLX5_ST_SZ_DW(create_virtio_q_counters_out)] = {}; 11831892a3d4SEli Cohen void *cmd_hdr; 11841892a3d4SEli Cohen int err; 11851892a3d4SEli Cohen 11861892a3d4SEli Cohen if (!counters_supported(&ndev->mvdev)) 11871892a3d4SEli Cohen return 0; 11881892a3d4SEli Cohen 11891892a3d4SEli Cohen cmd_hdr = MLX5_ADDR_OF(create_virtio_q_counters_in, in, hdr); 11901892a3d4SEli Cohen 11911892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 11921892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 11931892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 11941892a3d4SEli Cohen 11951892a3d4SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 11961892a3d4SEli Cohen if (err) 11971892a3d4SEli Cohen return err; 11981892a3d4SEli Cohen 11991892a3d4SEli Cohen mvq->counter_set_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 12001892a3d4SEli Cohen 12011892a3d4SEli Cohen return 0; 12021892a3d4SEli Cohen } 12031892a3d4SEli Cohen 12041892a3d4SEli Cohen static void counter_set_dealloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 12051892a3d4SEli Cohen { 12061892a3d4SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_virtio_q_counters_in)] = {}; 12071892a3d4SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_virtio_q_counters_out)] = {}; 12081892a3d4SEli Cohen 12091892a3d4SEli Cohen if (!counters_supported(&ndev->mvdev)) 12101892a3d4SEli Cohen return; 12111892a3d4SEli Cohen 12121892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 12131892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_id, mvq->counter_set_id); 12141892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.uid, ndev->mvdev.res.uid); 12151892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 12161892a3d4SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 12171892a3d4SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "dealloc counter set 0x%x\n", mvq->counter_set_id); 12181892a3d4SEli Cohen } 12191892a3d4SEli Cohen 12201a86b377SEli Cohen static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 12211a86b377SEli Cohen { 12221a86b377SEli Cohen u16 idx = mvq->index; 12231a86b377SEli Cohen int err; 12241a86b377SEli Cohen 12251a86b377SEli Cohen if (!mvq->num_ent) 12261a86b377SEli Cohen return 0; 12271a86b377SEli Cohen 122852893733SEli Cohen if (mvq->initialized) 122952893733SEli Cohen return 0; 12301a86b377SEli Cohen 12311a86b377SEli Cohen err = cq_create(ndev, idx, mvq->num_ent); 12321a86b377SEli Cohen if (err) 12331a86b377SEli Cohen return err; 12341a86b377SEli Cohen 12351a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->fwqp); 12361a86b377SEli Cohen if (err) 12371a86b377SEli Cohen goto err_fwqp; 12381a86b377SEli Cohen 12391a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->vqqp); 12401a86b377SEli Cohen if (err) 12411a86b377SEli Cohen goto err_vqqp; 12421a86b377SEli Cohen 12431a86b377SEli Cohen err = connect_qps(ndev, mvq); 12441a86b377SEli Cohen if (err) 12451a86b377SEli Cohen goto err_connect; 12461a86b377SEli Cohen 12471892a3d4SEli Cohen err = counter_set_alloc(ndev, mvq); 12481892a3d4SEli Cohen if (err) 12491892a3d4SEli Cohen goto err_counter; 12501892a3d4SEli Cohen 12511a86b377SEli Cohen err = create_virtqueue(ndev, mvq); 12521a86b377SEli Cohen if (err) 12531a86b377SEli Cohen goto err_connect; 12541a86b377SEli Cohen 12551a86b377SEli Cohen if (mvq->ready) { 12561a86b377SEli Cohen err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 12571a86b377SEli Cohen if (err) { 12581a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", 12591a86b377SEli Cohen idx, err); 12601a86b377SEli Cohen goto err_connect; 12611a86b377SEli Cohen } 12621a86b377SEli Cohen } 12631a86b377SEli Cohen 12641a86b377SEli Cohen mvq->initialized = true; 12651a86b377SEli Cohen return 0; 12661a86b377SEli Cohen 12671a86b377SEli Cohen err_connect: 12681892a3d4SEli Cohen counter_set_dealloc(ndev, mvq); 12691892a3d4SEli Cohen err_counter: 12701a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 12711a86b377SEli Cohen err_vqqp: 12721a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 12731a86b377SEli Cohen err_fwqp: 12741a86b377SEli Cohen cq_destroy(ndev, idx); 12751a86b377SEli Cohen return err; 12761a86b377SEli Cohen } 12771a86b377SEli Cohen 12781a86b377SEli Cohen static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 12791a86b377SEli Cohen { 12801a86b377SEli Cohen struct mlx5_virtq_attr attr; 12811a86b377SEli Cohen 12821a86b377SEli Cohen if (!mvq->initialized) 12831a86b377SEli Cohen return; 12841a86b377SEli Cohen 12851a86b377SEli Cohen if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 12861a86b377SEli Cohen return; 12871a86b377SEli Cohen 12881a86b377SEli Cohen if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) 12891a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); 12903176e974SSi-Wei Liu 12913176e974SSi-Wei Liu if (query_virtqueue(ndev, mvq, &attr)) { 12923176e974SSi-Wei Liu mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); 12933176e974SSi-Wei Liu return; 12943176e974SSi-Wei Liu } 12953176e974SSi-Wei Liu mvq->avail_idx = attr.available_index; 1296bc04d93eSEli Cohen mvq->used_idx = attr.used_index; 12971a86b377SEli Cohen } 12981a86b377SEli Cohen 12991a86b377SEli Cohen static void suspend_vqs(struct mlx5_vdpa_net *ndev) 13001a86b377SEli Cohen { 13011a86b377SEli Cohen int i; 13021a86b377SEli Cohen 130375560522SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 13041a86b377SEli Cohen suspend_vq(ndev, &ndev->vqs[i]); 13051a86b377SEli Cohen } 13061a86b377SEli Cohen 13071a86b377SEli Cohen static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 13081a86b377SEli Cohen { 13091a86b377SEli Cohen if (!mvq->initialized) 13101a86b377SEli Cohen return; 13111a86b377SEli Cohen 13121a86b377SEli Cohen suspend_vq(ndev, mvq); 13131a86b377SEli Cohen destroy_virtqueue(ndev, mvq); 13141892a3d4SEli Cohen counter_set_dealloc(ndev, mvq); 13151a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 13161a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 13171a86b377SEli Cohen cq_destroy(ndev, mvq->index); 13181a86b377SEli Cohen mvq->initialized = false; 13191a86b377SEli Cohen } 13201a86b377SEli Cohen 13211a86b377SEli Cohen static int create_rqt(struct mlx5_vdpa_net *ndev) 13221a86b377SEli Cohen { 1323a43ae805SEli Cohen int rqt_table_size = roundup_pow_of_two(ndev->rqt_size); 1324a43ae805SEli Cohen int act_sz = roundup_pow_of_two(ndev->cur_num_vqs / 2); 13251a86b377SEli Cohen __be32 *list; 13261a86b377SEli Cohen void *rqtc; 13271a86b377SEli Cohen int inlen; 13281a86b377SEli Cohen void *in; 13291a86b377SEli Cohen int i, j; 13301a86b377SEli Cohen int err; 13311a86b377SEli Cohen 1332a43ae805SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + rqt_table_size * MLX5_ST_SZ_BYTES(rq_num); 13331a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 13341a86b377SEli Cohen if (!in) 13351a86b377SEli Cohen return -ENOMEM; 13361a86b377SEli Cohen 13371a86b377SEli Cohen MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); 13381a86b377SEli Cohen rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 13391a86b377SEli Cohen 13401a86b377SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 1341a43ae805SEli Cohen MLX5_SET(rqtc, rqtc, rqt_max_size, rqt_table_size); 13421a86b377SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1343a43ae805SEli Cohen for (i = 0, j = 0; i < act_sz; i++, j += 2) 1344acde3929SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j % ndev->cur_num_vqs].virtq_id); 13451a86b377SEli Cohen 1346a43ae805SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz); 13471a86b377SEli Cohen err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); 13481a86b377SEli Cohen kfree(in); 13491a86b377SEli Cohen if (err) 13501a86b377SEli Cohen return err; 13511a86b377SEli Cohen 13521a86b377SEli Cohen return 0; 13531a86b377SEli Cohen } 13541a86b377SEli Cohen 135552893733SEli Cohen #define MLX5_MODIFY_RQT_NUM_RQS ((u64)1) 135652893733SEli Cohen 135752893733SEli Cohen static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) 135852893733SEli Cohen { 1359a43ae805SEli Cohen int act_sz = roundup_pow_of_two(num / 2); 136052893733SEli Cohen __be32 *list; 136152893733SEli Cohen void *rqtc; 136252893733SEli Cohen int inlen; 136352893733SEli Cohen void *in; 136452893733SEli Cohen int i, j; 136552893733SEli Cohen int err; 136652893733SEli Cohen 1367a43ae805SEli Cohen inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + act_sz * MLX5_ST_SZ_BYTES(rq_num); 136852893733SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 136952893733SEli Cohen if (!in) 137052893733SEli Cohen return -ENOMEM; 137152893733SEli Cohen 137252893733SEli Cohen MLX5_SET(modify_rqt_in, in, uid, ndev->mvdev.res.uid); 137352893733SEli Cohen MLX5_SET64(modify_rqt_in, in, bitmask, MLX5_MODIFY_RQT_NUM_RQS); 137452893733SEli Cohen rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 137552893733SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 137652893733SEli Cohen 137752893733SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1378a43ae805SEli Cohen for (i = 0, j = 0; i < act_sz; i++, j = j + 2) 1379a7f46ba4SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j % num].virtq_id); 138052893733SEli Cohen 1381a43ae805SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz); 138252893733SEli Cohen err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn); 138352893733SEli Cohen kfree(in); 138452893733SEli Cohen if (err) 138552893733SEli Cohen return err; 138652893733SEli Cohen 138752893733SEli Cohen return 0; 138852893733SEli Cohen } 138952893733SEli Cohen 13901a86b377SEli Cohen static void destroy_rqt(struct mlx5_vdpa_net *ndev) 13911a86b377SEli Cohen { 13921a86b377SEli Cohen mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); 13931a86b377SEli Cohen } 13941a86b377SEli Cohen 13951a86b377SEli Cohen static int create_tir(struct mlx5_vdpa_net *ndev) 13961a86b377SEli Cohen { 13971a86b377SEli Cohen #define HASH_IP_L4PORTS \ 13981a86b377SEli Cohen (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \ 13991a86b377SEli Cohen MLX5_HASH_FIELD_SEL_L4_DPORT) 14001a86b377SEli Cohen static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 14011a86b377SEli Cohen 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 14021a86b377SEli Cohen 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 14031a86b377SEli Cohen 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 14041a86b377SEli Cohen 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a }; 14051a86b377SEli Cohen void *rss_key; 14061a86b377SEli Cohen void *outer; 14071a86b377SEli Cohen void *tirc; 14081a86b377SEli Cohen void *in; 14091a86b377SEli Cohen int err; 14101a86b377SEli Cohen 14111a86b377SEli Cohen in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL); 14121a86b377SEli Cohen if (!in) 14131a86b377SEli Cohen return -ENOMEM; 14141a86b377SEli Cohen 14151a86b377SEli Cohen MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); 14161a86b377SEli Cohen tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 14171a86b377SEli Cohen MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 14181a86b377SEli Cohen 14191a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 14201a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ); 14211a86b377SEli Cohen rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 14221a86b377SEli Cohen memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key)); 14231a86b377SEli Cohen 14241a86b377SEli Cohen outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 14251a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); 14261a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); 14271a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS); 14281a86b377SEli Cohen 14291a86b377SEli Cohen MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); 14301a86b377SEli Cohen MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); 14311a86b377SEli Cohen 14321a86b377SEli Cohen err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); 14331a86b377SEli Cohen kfree(in); 14341a86b377SEli Cohen return err; 14351a86b377SEli Cohen } 14361a86b377SEli Cohen 14371a86b377SEli Cohen static void destroy_tir(struct mlx5_vdpa_net *ndev) 14381a86b377SEli Cohen { 14391a86b377SEli Cohen mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); 14401a86b377SEli Cohen } 14411a86b377SEli Cohen 1442baf2ad3fSEli Cohen #define MAX_STEERING_ENT 0x8000 1443baf2ad3fSEli Cohen #define MAX_STEERING_GROUPS 2 1444baf2ad3fSEli Cohen 1445baf2ad3fSEli Cohen static int mlx5_vdpa_add_mac_vlan_rules(struct mlx5_vdpa_net *ndev, u8 *mac, 1446baf2ad3fSEli Cohen u16 vid, bool tagged, 1447baf2ad3fSEli Cohen struct mlx5_flow_handle **ucast, 1448baf2ad3fSEli Cohen struct mlx5_flow_handle **mcast) 14491a86b377SEli Cohen { 14507becdd13SEli Cohen struct mlx5_flow_destination dest = {}; 14511a86b377SEli Cohen struct mlx5_flow_act flow_act = {}; 1452baf2ad3fSEli Cohen struct mlx5_flow_handle *rule; 1453540061acSEli Cohen struct mlx5_flow_spec *spec; 1454540061acSEli Cohen void *headers_c; 1455540061acSEli Cohen void *headers_v; 1456540061acSEli Cohen u8 *dmac_c; 1457540061acSEli Cohen u8 *dmac_v; 14581a86b377SEli Cohen int err; 14591a86b377SEli Cohen 1460540061acSEli Cohen spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1461540061acSEli Cohen if (!spec) 1462540061acSEli Cohen return -ENOMEM; 1463540061acSEli Cohen 1464540061acSEli Cohen spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1465540061acSEli Cohen headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers); 1466540061acSEli Cohen headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); 1467baf2ad3fSEli Cohen dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, outer_headers.dmac_47_16); 1468540061acSEli Cohen dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16); 146971aa95a6SXu Qiang eth_broadcast_addr(dmac_c); 1470baf2ad3fSEli Cohen ether_addr_copy(dmac_v, mac); 1471*a6ce72c0SEli Cohen if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VLAN)) { 1472baf2ad3fSEli Cohen MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1); 1473*a6ce72c0SEli Cohen MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, first_vid); 1474*a6ce72c0SEli Cohen } 1475baf2ad3fSEli Cohen if (tagged) { 1476baf2ad3fSEli Cohen MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); 1477*a6ce72c0SEli Cohen MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, vid); 1478baf2ad3fSEli Cohen } 14797becdd13SEli Cohen flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 14807becdd13SEli Cohen dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 14817becdd13SEli Cohen dest.tir_num = ndev->res.tirn; 1482baf2ad3fSEli Cohen rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, &dest, 1); 1483baf2ad3fSEli Cohen if (IS_ERR(rule)) 1484baf2ad3fSEli Cohen return PTR_ERR(rule); 1485540061acSEli Cohen 1486baf2ad3fSEli Cohen *ucast = rule; 14871a86b377SEli Cohen 1488540061acSEli Cohen memset(dmac_c, 0, ETH_ALEN); 1489540061acSEli Cohen memset(dmac_v, 0, ETH_ALEN); 1490540061acSEli Cohen dmac_c[0] = 1; 1491540061acSEli Cohen dmac_v[0] = 1; 1492baf2ad3fSEli Cohen rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, &dest, 1); 1493baf2ad3fSEli Cohen kvfree(spec); 1494baf2ad3fSEli Cohen if (IS_ERR(rule)) { 1495baf2ad3fSEli Cohen err = PTR_ERR(rule); 1496baf2ad3fSEli Cohen goto err_mcast; 1497540061acSEli Cohen } 1498540061acSEli Cohen 1499baf2ad3fSEli Cohen *mcast = rule; 15001a86b377SEli Cohen return 0; 15011a86b377SEli Cohen 1502baf2ad3fSEli Cohen err_mcast: 1503baf2ad3fSEli Cohen mlx5_del_flow_rules(*ucast); 15041a86b377SEli Cohen return err; 15051a86b377SEli Cohen } 15061a86b377SEli Cohen 1507baf2ad3fSEli Cohen static void mlx5_vdpa_del_mac_vlan_rules(struct mlx5_vdpa_net *ndev, 1508baf2ad3fSEli Cohen struct mlx5_flow_handle *ucast, 1509baf2ad3fSEli Cohen struct mlx5_flow_handle *mcast) 15101a86b377SEli Cohen { 1511baf2ad3fSEli Cohen mlx5_del_flow_rules(ucast); 1512baf2ad3fSEli Cohen mlx5_del_flow_rules(mcast); 1513baf2ad3fSEli Cohen } 1514baf2ad3fSEli Cohen 1515baf2ad3fSEli Cohen static u64 search_val(u8 *mac, u16 vlan, bool tagged) 1516baf2ad3fSEli Cohen { 1517baf2ad3fSEli Cohen u64 val; 1518baf2ad3fSEli Cohen 1519baf2ad3fSEli Cohen if (!tagged) 1520baf2ad3fSEli Cohen vlan = MLX5V_UNTAGGED; 1521baf2ad3fSEli Cohen 1522baf2ad3fSEli Cohen val = (u64)vlan << 48 | 1523baf2ad3fSEli Cohen (u64)mac[0] << 40 | 1524baf2ad3fSEli Cohen (u64)mac[1] << 32 | 1525baf2ad3fSEli Cohen (u64)mac[2] << 24 | 1526baf2ad3fSEli Cohen (u64)mac[3] << 16 | 1527baf2ad3fSEli Cohen (u64)mac[4] << 8 | 1528baf2ad3fSEli Cohen (u64)mac[5]; 1529baf2ad3fSEli Cohen 1530baf2ad3fSEli Cohen return val; 1531baf2ad3fSEli Cohen } 1532baf2ad3fSEli Cohen 1533baf2ad3fSEli Cohen static struct macvlan_node *mac_vlan_lookup(struct mlx5_vdpa_net *ndev, u64 value) 1534baf2ad3fSEli Cohen { 1535baf2ad3fSEli Cohen struct macvlan_node *pos; 1536baf2ad3fSEli Cohen u32 idx; 1537baf2ad3fSEli Cohen 1538baf2ad3fSEli Cohen idx = hash_64(value, 8); // tbd 8 1539baf2ad3fSEli Cohen hlist_for_each_entry(pos, &ndev->macvlan_hash[idx], hlist) { 1540baf2ad3fSEli Cohen if (pos->macvlan == value) 1541baf2ad3fSEli Cohen return pos; 1542baf2ad3fSEli Cohen } 1543baf2ad3fSEli Cohen return NULL; 1544baf2ad3fSEli Cohen } 1545baf2ad3fSEli Cohen 1546baf2ad3fSEli Cohen static int mac_vlan_add(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vlan, bool tagged) // vlan -> vid 1547baf2ad3fSEli Cohen { 1548baf2ad3fSEli Cohen struct macvlan_node *ptr; 1549baf2ad3fSEli Cohen u64 val; 1550baf2ad3fSEli Cohen u32 idx; 1551baf2ad3fSEli Cohen int err; 1552baf2ad3fSEli Cohen 1553baf2ad3fSEli Cohen val = search_val(mac, vlan, tagged); 1554baf2ad3fSEli Cohen if (mac_vlan_lookup(ndev, val)) 1555baf2ad3fSEli Cohen return -EEXIST; 1556baf2ad3fSEli Cohen 1557baf2ad3fSEli Cohen ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); 1558baf2ad3fSEli Cohen if (!ptr) 1559baf2ad3fSEli Cohen return -ENOMEM; 1560baf2ad3fSEli Cohen 1561baf2ad3fSEli Cohen err = mlx5_vdpa_add_mac_vlan_rules(ndev, ndev->config.mac, vlan, tagged, 1562baf2ad3fSEli Cohen &ptr->ucast_rule, &ptr->mcast_rule); 1563baf2ad3fSEli Cohen if (err) 1564baf2ad3fSEli Cohen goto err_add; 1565baf2ad3fSEli Cohen 1566baf2ad3fSEli Cohen ptr->macvlan = val; 1567baf2ad3fSEli Cohen idx = hash_64(val, 8); 1568baf2ad3fSEli Cohen hlist_add_head(&ptr->hlist, &ndev->macvlan_hash[idx]); 1569baf2ad3fSEli Cohen return 0; 1570baf2ad3fSEli Cohen 1571baf2ad3fSEli Cohen err_add: 1572baf2ad3fSEli Cohen kfree(ptr); 1573baf2ad3fSEli Cohen return err; 1574baf2ad3fSEli Cohen } 1575baf2ad3fSEli Cohen 1576baf2ad3fSEli Cohen static void mac_vlan_del(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vlan, bool tagged) 1577baf2ad3fSEli Cohen { 1578baf2ad3fSEli Cohen struct macvlan_node *ptr; 1579baf2ad3fSEli Cohen 1580baf2ad3fSEli Cohen ptr = mac_vlan_lookup(ndev, search_val(mac, vlan, tagged)); 1581baf2ad3fSEli Cohen if (!ptr) 15821a86b377SEli Cohen return; 15831a86b377SEli Cohen 1584baf2ad3fSEli Cohen hlist_del(&ptr->hlist); 1585baf2ad3fSEli Cohen mlx5_vdpa_del_mac_vlan_rules(ndev, ptr->ucast_rule, ptr->mcast_rule); 1586baf2ad3fSEli Cohen kfree(ptr); 1587baf2ad3fSEli Cohen } 1588baf2ad3fSEli Cohen 1589baf2ad3fSEli Cohen static void clear_mac_vlan_table(struct mlx5_vdpa_net *ndev) 1590baf2ad3fSEli Cohen { 1591baf2ad3fSEli Cohen struct macvlan_node *pos; 1592baf2ad3fSEli Cohen struct hlist_node *n; 1593baf2ad3fSEli Cohen int i; 1594baf2ad3fSEli Cohen 1595baf2ad3fSEli Cohen for (i = 0; i < MLX5V_MACVLAN_SIZE; i++) { 1596baf2ad3fSEli Cohen hlist_for_each_entry_safe(pos, n, &ndev->macvlan_hash[i], hlist) { 1597baf2ad3fSEli Cohen hlist_del(&pos->hlist); 1598baf2ad3fSEli Cohen mlx5_vdpa_del_mac_vlan_rules(ndev, pos->ucast_rule, pos->mcast_rule); 1599baf2ad3fSEli Cohen kfree(pos); 1600baf2ad3fSEli Cohen } 1601baf2ad3fSEli Cohen } 1602baf2ad3fSEli Cohen } 1603baf2ad3fSEli Cohen 1604baf2ad3fSEli Cohen static int setup_steering(struct mlx5_vdpa_net *ndev) 1605baf2ad3fSEli Cohen { 1606baf2ad3fSEli Cohen struct mlx5_flow_table_attr ft_attr = {}; 1607baf2ad3fSEli Cohen struct mlx5_flow_namespace *ns; 1608baf2ad3fSEli Cohen int err; 1609baf2ad3fSEli Cohen 1610baf2ad3fSEli Cohen ft_attr.max_fte = MAX_STEERING_ENT; 1611baf2ad3fSEli Cohen ft_attr.autogroup.max_num_groups = MAX_STEERING_GROUPS; 1612baf2ad3fSEli Cohen 1613baf2ad3fSEli Cohen ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); 1614baf2ad3fSEli Cohen if (!ns) { 1615baf2ad3fSEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to get flow namespace\n"); 1616baf2ad3fSEli Cohen return -EOPNOTSUPP; 1617baf2ad3fSEli Cohen } 1618baf2ad3fSEli Cohen 1619baf2ad3fSEli Cohen ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 1620baf2ad3fSEli Cohen if (IS_ERR(ndev->rxft)) { 1621baf2ad3fSEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to create flow table\n"); 1622baf2ad3fSEli Cohen return PTR_ERR(ndev->rxft); 1623baf2ad3fSEli Cohen } 1624baf2ad3fSEli Cohen 1625baf2ad3fSEli Cohen err = mac_vlan_add(ndev, ndev->config.mac, 0, false); 1626baf2ad3fSEli Cohen if (err) 1627baf2ad3fSEli Cohen goto err_add; 1628baf2ad3fSEli Cohen 1629baf2ad3fSEli Cohen return 0; 1630baf2ad3fSEli Cohen 1631baf2ad3fSEli Cohen err_add: 1632baf2ad3fSEli Cohen mlx5_destroy_flow_table(ndev->rxft); 1633baf2ad3fSEli Cohen return err; 1634baf2ad3fSEli Cohen } 1635baf2ad3fSEli Cohen 1636baf2ad3fSEli Cohen static void teardown_steering(struct mlx5_vdpa_net *ndev) 1637baf2ad3fSEli Cohen { 1638baf2ad3fSEli Cohen clear_mac_vlan_table(ndev); 16391a86b377SEli Cohen mlx5_destroy_flow_table(ndev->rxft); 16401a86b377SEli Cohen } 16411a86b377SEli Cohen 16425262912eSEli Cohen static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) 16435262912eSEli Cohen { 16445262912eSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 16455262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 16465262912eSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 16475262912eSEli Cohen struct mlx5_core_dev *pfmdev; 16485262912eSEli Cohen size_t read; 1649f1781bedSMichael Qiu u8 mac[ETH_ALEN], mac_back[ETH_ALEN]; 16505262912eSEli Cohen 16515262912eSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 16525262912eSEli Cohen switch (cmd) { 16535262912eSEli Cohen case VIRTIO_NET_CTRL_MAC_ADDR_SET: 16545262912eSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)mac, ETH_ALEN); 16555262912eSEli Cohen if (read != ETH_ALEN) 16565262912eSEli Cohen break; 16575262912eSEli Cohen 16585262912eSEli Cohen if (!memcmp(ndev->config.mac, mac, 6)) { 16595262912eSEli Cohen status = VIRTIO_NET_OK; 16605262912eSEli Cohen break; 16615262912eSEli Cohen } 16625262912eSEli Cohen 1663f1781bedSMichael Qiu if (is_zero_ether_addr(mac)) 1664f1781bedSMichael Qiu break; 1665f1781bedSMichael Qiu 16665262912eSEli Cohen if (!is_zero_ether_addr(ndev->config.mac)) { 16675262912eSEli Cohen if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 16685262912eSEli Cohen mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n", 16695262912eSEli Cohen ndev->config.mac); 16705262912eSEli Cohen break; 16715262912eSEli Cohen } 16725262912eSEli Cohen } 16735262912eSEli Cohen 16745262912eSEli Cohen if (mlx5_mpfs_add_mac(pfmdev, mac)) { 16755262912eSEli Cohen mlx5_vdpa_warn(mvdev, "failed to insert new MAC %pM into MPFS table\n", 16765262912eSEli Cohen mac); 16775262912eSEli Cohen break; 16785262912eSEli Cohen } 16795262912eSEli Cohen 1680f1781bedSMichael Qiu /* backup the original mac address so that if failed to add the forward rules 1681f1781bedSMichael Qiu * we could restore it 1682f1781bedSMichael Qiu */ 1683f1781bedSMichael Qiu memcpy(mac_back, ndev->config.mac, ETH_ALEN); 1684f1781bedSMichael Qiu 16855262912eSEli Cohen memcpy(ndev->config.mac, mac, ETH_ALEN); 1686f1781bedSMichael Qiu 1687f1781bedSMichael Qiu /* Need recreate the flow table entry, so that the packet could forward back 1688f1781bedSMichael Qiu */ 1689baf2ad3fSEli Cohen mac_vlan_del(ndev, ndev->config.mac, 0, false); 1690f1781bedSMichael Qiu 1691baf2ad3fSEli Cohen if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) { 1692f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "failed to insert forward rules, try to restore\n"); 1693f1781bedSMichael Qiu 1694f1781bedSMichael Qiu /* Although it hardly run here, we still need double check */ 1695f1781bedSMichael Qiu if (is_zero_ether_addr(mac_back)) { 1696f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore mac failed: Original MAC is zero\n"); 1697f1781bedSMichael Qiu break; 1698f1781bedSMichael Qiu } 1699f1781bedSMichael Qiu 1700f1781bedSMichael Qiu /* Try to restore original mac address to MFPS table, and try to restore 1701f1781bedSMichael Qiu * the forward rule entry. 1702f1781bedSMichael Qiu */ 1703f1781bedSMichael Qiu if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 1704f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore mac failed: delete MAC %pM from MPFS table failed\n", 1705f1781bedSMichael Qiu ndev->config.mac); 1706f1781bedSMichael Qiu } 1707f1781bedSMichael Qiu 1708f1781bedSMichael Qiu if (mlx5_mpfs_add_mac(pfmdev, mac_back)) { 1709f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore mac failed: insert old MAC %pM into MPFS table failed\n", 1710f1781bedSMichael Qiu mac_back); 1711f1781bedSMichael Qiu } 1712f1781bedSMichael Qiu 1713f1781bedSMichael Qiu memcpy(ndev->config.mac, mac_back, ETH_ALEN); 1714f1781bedSMichael Qiu 1715baf2ad3fSEli Cohen if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) 1716f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore forward rules failed: insert forward rules failed\n"); 1717f1781bedSMichael Qiu 1718f1781bedSMichael Qiu break; 1719f1781bedSMichael Qiu } 1720f1781bedSMichael Qiu 17215262912eSEli Cohen status = VIRTIO_NET_OK; 17225262912eSEli Cohen break; 17235262912eSEli Cohen 17245262912eSEli Cohen default: 17255262912eSEli Cohen break; 17265262912eSEli Cohen } 17275262912eSEli Cohen 17285262912eSEli Cohen return status; 17295262912eSEli Cohen } 17305262912eSEli Cohen 173152893733SEli Cohen static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps) 173252893733SEli Cohen { 173352893733SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 173452893733SEli Cohen int cur_qps = ndev->cur_num_vqs / 2; 173552893733SEli Cohen int err; 173652893733SEli Cohen int i; 173752893733SEli Cohen 173852893733SEli Cohen if (cur_qps > newqps) { 173952893733SEli Cohen err = modify_rqt(ndev, 2 * newqps); 174052893733SEli Cohen if (err) 174152893733SEli Cohen return err; 174252893733SEli Cohen 174352893733SEli Cohen for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) 174452893733SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 174552893733SEli Cohen 174652893733SEli Cohen ndev->cur_num_vqs = 2 * newqps; 174752893733SEli Cohen } else { 174852893733SEli Cohen ndev->cur_num_vqs = 2 * newqps; 174952893733SEli Cohen for (i = cur_qps * 2; i < 2 * newqps; i++) { 175052893733SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 175152893733SEli Cohen if (err) 175252893733SEli Cohen goto clean_added; 175352893733SEli Cohen } 175452893733SEli Cohen err = modify_rqt(ndev, 2 * newqps); 175552893733SEli Cohen if (err) 175652893733SEli Cohen goto clean_added; 175752893733SEli Cohen } 175852893733SEli Cohen return 0; 175952893733SEli Cohen 176052893733SEli Cohen clean_added: 176137e07e70SEli Cohen for (--i; i >= 2 * cur_qps; --i) 176252893733SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 176352893733SEli Cohen 176437e07e70SEli Cohen ndev->cur_num_vqs = 2 * cur_qps; 176537e07e70SEli Cohen 176652893733SEli Cohen return err; 176752893733SEli Cohen } 176852893733SEli Cohen 176952893733SEli Cohen static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd) 177052893733SEli Cohen { 177152893733SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 177252893733SEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 177352893733SEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 177452893733SEli Cohen struct virtio_net_ctrl_mq mq; 177552893733SEli Cohen size_t read; 177652893733SEli Cohen u16 newqps; 177752893733SEli Cohen 177852893733SEli Cohen switch (cmd) { 177952893733SEli Cohen case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: 1780ed0f849fSSi-Wei Liu /* This mq feature check aligns with pre-existing userspace 1781ed0f849fSSi-Wei Liu * implementation. 1782ed0f849fSSi-Wei Liu * 1783ed0f849fSSi-Wei Liu * Without it, an untrusted driver could fake a multiqueue config 1784ed0f849fSSi-Wei Liu * request down to a non-mq device that may cause kernel to 1785ed0f849fSSi-Wei Liu * panic due to uninitialized resources for extra vqs. Even with 1786ed0f849fSSi-Wei Liu * a well behaving guest driver, it is not expected to allow 1787ed0f849fSSi-Wei Liu * changing the number of vqs on a non-mq device. 1788ed0f849fSSi-Wei Liu */ 1789ed0f849fSSi-Wei Liu if (!MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) 1790ed0f849fSSi-Wei Liu break; 1791ed0f849fSSi-Wei Liu 179252893733SEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)&mq, sizeof(mq)); 179352893733SEli Cohen if (read != sizeof(mq)) 179452893733SEli Cohen break; 179552893733SEli Cohen 179652893733SEli Cohen newqps = mlx5vdpa16_to_cpu(mvdev, mq.virtqueue_pairs); 1797ed0f849fSSi-Wei Liu if (newqps < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 1798acde3929SEli Cohen newqps > ndev->rqt_size) 1799ed0f849fSSi-Wei Liu break; 1800ed0f849fSSi-Wei Liu 180152893733SEli Cohen if (ndev->cur_num_vqs == 2 * newqps) { 180252893733SEli Cohen status = VIRTIO_NET_OK; 180352893733SEli Cohen break; 180452893733SEli Cohen } 180552893733SEli Cohen 180652893733SEli Cohen if (!change_num_qps(mvdev, newqps)) 180752893733SEli Cohen status = VIRTIO_NET_OK; 180852893733SEli Cohen 180952893733SEli Cohen break; 181052893733SEli Cohen default: 181152893733SEli Cohen break; 181252893733SEli Cohen } 181352893733SEli Cohen 181452893733SEli Cohen return status; 181552893733SEli Cohen } 181652893733SEli Cohen 1817baf2ad3fSEli Cohen static virtio_net_ctrl_ack handle_ctrl_vlan(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1818baf2ad3fSEli Cohen { 1819baf2ad3fSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1820baf2ad3fSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1821baf2ad3fSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 1822baf2ad3fSEli Cohen __virtio16 vlan; 1823baf2ad3fSEli Cohen size_t read; 1824baf2ad3fSEli Cohen u16 id; 1825baf2ad3fSEli Cohen 1826baf2ad3fSEli Cohen switch (cmd) { 1827baf2ad3fSEli Cohen case VIRTIO_NET_CTRL_VLAN_ADD: 1828baf2ad3fSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &vlan, sizeof(vlan)); 1829baf2ad3fSEli Cohen if (read != sizeof(vlan)) 1830baf2ad3fSEli Cohen break; 1831baf2ad3fSEli Cohen 1832baf2ad3fSEli Cohen id = mlx5vdpa16_to_cpu(mvdev, vlan); 1833baf2ad3fSEli Cohen if (mac_vlan_add(ndev, ndev->config.mac, id, true)) 1834baf2ad3fSEli Cohen break; 1835baf2ad3fSEli Cohen 1836baf2ad3fSEli Cohen status = VIRTIO_NET_OK; 1837baf2ad3fSEli Cohen break; 1838baf2ad3fSEli Cohen case VIRTIO_NET_CTRL_VLAN_DEL: 1839baf2ad3fSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &vlan, sizeof(vlan)); 1840baf2ad3fSEli Cohen if (read != sizeof(vlan)) 1841baf2ad3fSEli Cohen break; 1842baf2ad3fSEli Cohen 1843baf2ad3fSEli Cohen id = mlx5vdpa16_to_cpu(mvdev, vlan); 1844baf2ad3fSEli Cohen mac_vlan_del(ndev, ndev->config.mac, id, true); 1845f766c409SDan Carpenter status = VIRTIO_NET_OK; 1846baf2ad3fSEli Cohen break; 1847baf2ad3fSEli Cohen default: 1848baf2ad3fSEli Cohen break; 1849baf2ad3fSEli Cohen } 1850baf2ad3fSEli Cohen 1851baf2ad3fSEli Cohen return status; 1852baf2ad3fSEli Cohen } 1853baf2ad3fSEli Cohen 18545262912eSEli Cohen static void mlx5_cvq_kick_handler(struct work_struct *work) 18555262912eSEli Cohen { 18565262912eSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 18575262912eSEli Cohen struct virtio_net_ctrl_hdr ctrl; 1858218bdd20SEli Cohen struct mlx5_vdpa_wq_ent *wqent; 18595262912eSEli Cohen struct mlx5_vdpa_dev *mvdev; 18605262912eSEli Cohen struct mlx5_control_vq *cvq; 18615262912eSEli Cohen struct mlx5_vdpa_net *ndev; 18625262912eSEli Cohen size_t read, write; 18635262912eSEli Cohen int err; 18645262912eSEli Cohen 1865218bdd20SEli Cohen wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 18665262912eSEli Cohen mvdev = wqent->mvdev; 18675262912eSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 18685262912eSEli Cohen cvq = &mvdev->cvq; 18691c80cf03SJason Wang 1870759ae7f9SEli Cohen down_write(&ndev->reslock); 18711c80cf03SJason Wang 18721c80cf03SJason Wang if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 18731c80cf03SJason Wang goto out; 18741c80cf03SJason Wang 18755262912eSEli Cohen if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 18761c80cf03SJason Wang goto out; 18775262912eSEli Cohen 18785262912eSEli Cohen if (!cvq->ready) 18791c80cf03SJason Wang goto out; 18805262912eSEli Cohen 18815262912eSEli Cohen while (true) { 18825262912eSEli Cohen err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head, 18835262912eSEli Cohen GFP_ATOMIC); 18845262912eSEli Cohen if (err <= 0) 18855262912eSEli Cohen break; 18865262912eSEli Cohen 18875262912eSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &ctrl, sizeof(ctrl)); 18885262912eSEli Cohen if (read != sizeof(ctrl)) 18895262912eSEli Cohen break; 18905262912eSEli Cohen 18911892a3d4SEli Cohen cvq->received_desc++; 18925262912eSEli Cohen switch (ctrl.class) { 18935262912eSEli Cohen case VIRTIO_NET_CTRL_MAC: 18945262912eSEli Cohen status = handle_ctrl_mac(mvdev, ctrl.cmd); 18955262912eSEli Cohen break; 189652893733SEli Cohen case VIRTIO_NET_CTRL_MQ: 189752893733SEli Cohen status = handle_ctrl_mq(mvdev, ctrl.cmd); 189852893733SEli Cohen break; 1899baf2ad3fSEli Cohen case VIRTIO_NET_CTRL_VLAN: 1900baf2ad3fSEli Cohen status = handle_ctrl_vlan(mvdev, ctrl.cmd); 1901baf2ad3fSEli Cohen break; 19025262912eSEli Cohen default: 19035262912eSEli Cohen break; 19045262912eSEli Cohen } 19055262912eSEli Cohen 19065262912eSEli Cohen /* Make sure data is written before advancing index */ 19075262912eSEli Cohen smp_wmb(); 19085262912eSEli Cohen 19095262912eSEli Cohen write = vringh_iov_push_iotlb(&cvq->vring, &cvq->wiov, &status, sizeof(status)); 19105262912eSEli Cohen vringh_complete_iotlb(&cvq->vring, cvq->head, write); 19115262912eSEli Cohen vringh_kiov_cleanup(&cvq->riov); 19125262912eSEli Cohen vringh_kiov_cleanup(&cvq->wiov); 19135262912eSEli Cohen 19145262912eSEli Cohen if (vringh_need_notify_iotlb(&cvq->vring)) 19155262912eSEli Cohen vringh_notify(&cvq->vring); 191655ebf0d6SJason Wang 19171892a3d4SEli Cohen cvq->completed_desc++; 191855ebf0d6SJason Wang queue_work(mvdev->wq, &wqent->work); 191955ebf0d6SJason Wang break; 19205262912eSEli Cohen } 19211c80cf03SJason Wang 19221c80cf03SJason Wang out: 1923759ae7f9SEli Cohen up_write(&ndev->reslock); 19245262912eSEli Cohen } 19255262912eSEli Cohen 19261a86b377SEli Cohen static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) 19271a86b377SEli Cohen { 19281a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19291a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19305262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 19311a86b377SEli Cohen 1932e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1933e4fc6650SEli Cohen return; 1934e4fc6650SEli Cohen 19355262912eSEli Cohen if (unlikely(is_ctrl_vq_idx(mvdev, idx))) { 1936ad6dc1daSEli Cohen if (!mvdev->wq || !mvdev->cvq.ready) 19375262912eSEli Cohen return; 19385262912eSEli Cohen 193955ebf0d6SJason Wang queue_work(mvdev->wq, &ndev->cvq_ent.work); 19405262912eSEli Cohen return; 19415262912eSEli Cohen } 19425262912eSEli Cohen 19435262912eSEli Cohen mvq = &ndev->vqs[idx]; 19441a86b377SEli Cohen if (unlikely(!mvq->ready)) 19451a86b377SEli Cohen return; 19461a86b377SEli Cohen 19471a86b377SEli Cohen iowrite16(idx, ndev->mvdev.res.kick_addr); 19481a86b377SEli Cohen } 19491a86b377SEli Cohen 19501a86b377SEli Cohen static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 19511a86b377SEli Cohen u64 driver_area, u64 device_area) 19521a86b377SEli Cohen { 19531a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19541a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19555262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 19561a86b377SEli Cohen 1957e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1958e4fc6650SEli Cohen return -EINVAL; 1959e4fc6650SEli Cohen 19605262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 19615262912eSEli Cohen mvdev->cvq.desc_addr = desc_area; 19625262912eSEli Cohen mvdev->cvq.device_addr = device_area; 19635262912eSEli Cohen mvdev->cvq.driver_addr = driver_area; 19645262912eSEli Cohen return 0; 19655262912eSEli Cohen } 19665262912eSEli Cohen 19675262912eSEli Cohen mvq = &ndev->vqs[idx]; 19681a86b377SEli Cohen mvq->desc_addr = desc_area; 19691a86b377SEli Cohen mvq->device_addr = device_area; 19701a86b377SEli Cohen mvq->driver_addr = driver_area; 19711a86b377SEli Cohen return 0; 19721a86b377SEli Cohen } 19731a86b377SEli Cohen 19741a86b377SEli Cohen static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 19751a86b377SEli Cohen { 19761a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19771a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19781a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 19791a86b377SEli Cohen 19805262912eSEli Cohen if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 1981e4fc6650SEli Cohen return; 1982e4fc6650SEli Cohen 19831a86b377SEli Cohen mvq = &ndev->vqs[idx]; 19841a86b377SEli Cohen mvq->num_ent = num; 19851a86b377SEli Cohen } 19861a86b377SEli Cohen 19871a86b377SEli Cohen static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 19881a86b377SEli Cohen { 19891a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19901a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19911a86b377SEli Cohen 1992db296d25SEli Cohen ndev->event_cbs[idx] = *cb; 199340f2f3e9SEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) 199440f2f3e9SEli Cohen mvdev->cvq.event_cb = *cb; 19951a86b377SEli Cohen } 19961a86b377SEli Cohen 19975262912eSEli Cohen static void mlx5_cvq_notify(struct vringh *vring) 19985262912eSEli Cohen { 19995262912eSEli Cohen struct mlx5_control_vq *cvq = container_of(vring, struct mlx5_control_vq, vring); 20005262912eSEli Cohen 20015262912eSEli Cohen if (!cvq->event_cb.callback) 20025262912eSEli Cohen return; 20035262912eSEli Cohen 20045262912eSEli Cohen cvq->event_cb.callback(cvq->event_cb.private); 20055262912eSEli Cohen } 20065262912eSEli Cohen 20075262912eSEli Cohen static void set_cvq_ready(struct mlx5_vdpa_dev *mvdev, bool ready) 20085262912eSEli Cohen { 20095262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 20105262912eSEli Cohen 20115262912eSEli Cohen cvq->ready = ready; 20125262912eSEli Cohen if (!ready) 20135262912eSEli Cohen return; 20145262912eSEli Cohen 20155262912eSEli Cohen cvq->vring.notify = mlx5_cvq_notify; 20161a86b377SEli Cohen } 20171a86b377SEli Cohen 20181a86b377SEli Cohen static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 20191a86b377SEli Cohen { 20201a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 20211a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20225262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 2023cae15c2eSEli Cohen int err; 20241a86b377SEli Cohen 2025759be899SEli Cohen if (!mvdev->actual_features) 2026759be899SEli Cohen return; 2027759be899SEli Cohen 2028e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 2029e4fc6650SEli Cohen return; 2030e4fc6650SEli Cohen 20315262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 20325262912eSEli Cohen set_cvq_ready(mvdev, ready); 20335262912eSEli Cohen return; 20345262912eSEli Cohen } 20355262912eSEli Cohen 20365262912eSEli Cohen mvq = &ndev->vqs[idx]; 2037cae15c2eSEli Cohen if (!ready) { 20381a86b377SEli Cohen suspend_vq(ndev, mvq); 2039cae15c2eSEli Cohen } else { 2040cae15c2eSEli Cohen err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 2041cae15c2eSEli Cohen if (err) { 2042cae15c2eSEli Cohen mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err); 2043cae15c2eSEli Cohen ready = false; 2044cae15c2eSEli Cohen } 2045cae15c2eSEli Cohen } 2046cae15c2eSEli Cohen 20471a86b377SEli Cohen 20481a86b377SEli Cohen mvq->ready = ready; 20491a86b377SEli Cohen } 20501a86b377SEli Cohen 20511a86b377SEli Cohen static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx) 20521a86b377SEli Cohen { 20531a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 20541a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20551a86b377SEli Cohen 2056e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 2057e4fc6650SEli Cohen return false; 2058e4fc6650SEli Cohen 20595262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) 20605262912eSEli Cohen return mvdev->cvq.ready; 20615262912eSEli Cohen 20625262912eSEli Cohen return ndev->vqs[idx].ready; 20631a86b377SEli Cohen } 20641a86b377SEli Cohen 20651a86b377SEli Cohen static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx, 20661a86b377SEli Cohen const struct vdpa_vq_state *state) 20671a86b377SEli Cohen { 20681a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 20691a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20705262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 20711a86b377SEli Cohen 2072e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 2073e4fc6650SEli Cohen return -EINVAL; 2074e4fc6650SEli Cohen 20755262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 20765262912eSEli Cohen mvdev->cvq.vring.last_avail_idx = state->split.avail_index; 20775262912eSEli Cohen return 0; 20785262912eSEli Cohen } 20795262912eSEli Cohen 20805262912eSEli Cohen mvq = &ndev->vqs[idx]; 20811a86b377SEli Cohen if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) { 20821a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "can't modify available index\n"); 20831a86b377SEli Cohen return -EINVAL; 20841a86b377SEli Cohen } 20851a86b377SEli Cohen 2086530a5678SJason Wang mvq->used_idx = state->split.avail_index; 2087530a5678SJason Wang mvq->avail_idx = state->split.avail_index; 20881a86b377SEli Cohen return 0; 20891a86b377SEli Cohen } 20901a86b377SEli Cohen 20911a86b377SEli Cohen static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 20921a86b377SEli Cohen { 20931a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 20941a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20955262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 20961a86b377SEli Cohen struct mlx5_virtq_attr attr; 20971a86b377SEli Cohen int err; 20981a86b377SEli Cohen 2099e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 2100e4fc6650SEli Cohen return -EINVAL; 2101e4fc6650SEli Cohen 21025262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 21035262912eSEli Cohen state->split.avail_index = mvdev->cvq.vring.last_avail_idx; 21045262912eSEli Cohen return 0; 21055262912eSEli Cohen } 21065262912eSEli Cohen 21075262912eSEli Cohen mvq = &ndev->vqs[idx]; 21083176e974SSi-Wei Liu /* If the virtq object was destroyed, use the value saved at 21093176e974SSi-Wei Liu * the last minute of suspend_vq. This caters for userspace 21103176e974SSi-Wei Liu * that cares about emulating the index after vq is stopped. 21113176e974SSi-Wei Liu */ 21123176e974SSi-Wei Liu if (!mvq->initialized) { 2113bc04d93eSEli Cohen /* Firmware returns a wrong value for the available index. 2114bc04d93eSEli Cohen * Since both values should be identical, we take the value of 2115bc04d93eSEli Cohen * used_idx which is reported correctly. 2116bc04d93eSEli Cohen */ 2117530a5678SJason Wang state->split.avail_index = mvq->used_idx; 21183176e974SSi-Wei Liu return 0; 21193176e974SSi-Wei Liu } 21201a86b377SEli Cohen 21211a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 21221a86b377SEli Cohen if (err) { 21231a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n"); 21241a86b377SEli Cohen return err; 21251a86b377SEli Cohen } 2126530a5678SJason Wang state->split.avail_index = attr.used_index; 21271a86b377SEli Cohen return 0; 21281a86b377SEli Cohen } 21291a86b377SEli Cohen 21301a86b377SEli Cohen static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) 21311a86b377SEli Cohen { 21321a86b377SEli Cohen return PAGE_SIZE; 21331a86b377SEli Cohen } 21341a86b377SEli Cohen 21358fcd20c3SEli Cohen static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx) 2136d4821902SGautam Dawar { 21378fcd20c3SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 21388fcd20c3SEli Cohen 21398fcd20c3SEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) 21408fcd20c3SEli Cohen return MLX5_VDPA_CVQ_GROUP; 21418fcd20c3SEli Cohen 21428fcd20c3SEli Cohen return MLX5_VDPA_DATAVQ_GROUP; 2143d4821902SGautam Dawar } 2144d4821902SGautam Dawar 21451a86b377SEli Cohen enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9, 21461a86b377SEli Cohen MLX5_VIRTIO_NET_F_CSUM = 1 << 10, 21471a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11, 21481a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12, 21491a86b377SEli Cohen }; 21501a86b377SEli Cohen 21511a86b377SEli Cohen static u64 mlx_to_vritio_features(u16 dev_features) 21521a86b377SEli Cohen { 21531a86b377SEli Cohen u64 result = 0; 21541a86b377SEli Cohen 21551a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM) 2156cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM); 21571a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_CSUM) 2158cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_CSUM); 21591a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6) 2160cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6); 21611a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4) 2162cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4); 21631a86b377SEli Cohen 21641a86b377SEli Cohen return result; 21651a86b377SEli Cohen } 21661a86b377SEli Cohen 216779de65edSEli Cohen static u64 get_supported_features(struct mlx5_core_dev *mdev) 216879de65edSEli Cohen { 216979de65edSEli Cohen u64 mlx_vdpa_features = 0; 217079de65edSEli Cohen u16 dev_features; 217179de65edSEli Cohen 217279de65edSEli Cohen dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mdev, device_features_bits_mask); 217379de65edSEli Cohen mlx_vdpa_features |= mlx_to_vritio_features(dev_features); 217479de65edSEli Cohen if (MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_version_1_0)) 217579de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_F_VERSION_1); 217679de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM); 217779de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VQ); 217879de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR); 217979de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MQ); 218079de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_STATUS); 218179de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MTU); 2182baf2ad3fSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VLAN); 218379de65edSEli Cohen 218479de65edSEli Cohen return mlx_vdpa_features; 218579de65edSEli Cohen } 218679de65edSEli Cohen 2187a64917bcSEli Cohen static u64 mlx5_vdpa_get_device_features(struct vdpa_device *vdev) 21881a86b377SEli Cohen { 21891a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 21901a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 21915262912eSEli Cohen 21921a86b377SEli Cohen print_features(mvdev, ndev->mvdev.mlx_features, false); 21931a86b377SEli Cohen return ndev->mvdev.mlx_features; 21941a86b377SEli Cohen } 21951a86b377SEli Cohen 219630c22f38SSi-Wei Liu static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features) 21971a86b377SEli Cohen { 219830c22f38SSi-Wei Liu /* Minimum features to expect */ 2199cbb52359SNathan Chancellor if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) 22001a86b377SEli Cohen return -EOPNOTSUPP; 22011a86b377SEli Cohen 220230c22f38SSi-Wei Liu /* Double check features combination sent down by the driver. 220330c22f38SSi-Wei Liu * Fail invalid features due to absence of the depended feature. 220430c22f38SSi-Wei Liu * 220530c22f38SSi-Wei Liu * Per VIRTIO v1.1 specification, section 5.1.3.1 Feature bit 220630c22f38SSi-Wei Liu * requirements: "VIRTIO_NET_F_MQ Requires VIRTIO_NET_F_CTRL_VQ". 220730c22f38SSi-Wei Liu * By failing the invalid features sent down by untrusted drivers, 220830c22f38SSi-Wei Liu * we're assured the assumption made upon is_index_valid() and 220930c22f38SSi-Wei Liu * is_ctrl_vq_idx() will not be compromised. 221030c22f38SSi-Wei Liu */ 221130c22f38SSi-Wei Liu if ((features & (BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) == 221230c22f38SSi-Wei Liu BIT_ULL(VIRTIO_NET_F_MQ)) 221330c22f38SSi-Wei Liu return -EINVAL; 221430c22f38SSi-Wei Liu 22151a86b377SEli Cohen return 0; 22161a86b377SEli Cohen } 22171a86b377SEli Cohen 2218ae0428deSEli Cohen static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev) 22191a86b377SEli Cohen { 2220ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 22211a86b377SEli Cohen int err; 22221a86b377SEli Cohen int i; 22231a86b377SEli Cohen 2224acde3929SEli Cohen for (i = 0; i < mvdev->max_vqs; i++) { 22251a86b377SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 22261a86b377SEli Cohen if (err) 22271a86b377SEli Cohen goto err_vq; 22281a86b377SEli Cohen } 22291a86b377SEli Cohen 22301a86b377SEli Cohen return 0; 22311a86b377SEli Cohen 22321a86b377SEli Cohen err_vq: 22331a86b377SEli Cohen for (--i; i >= 0; i--) 22341a86b377SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 22351a86b377SEli Cohen 22361a86b377SEli Cohen return err; 22371a86b377SEli Cohen } 22381a86b377SEli Cohen 22391a86b377SEli Cohen static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) 22401a86b377SEli Cohen { 22411a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 22421a86b377SEli Cohen int i; 22431a86b377SEli Cohen 22441a86b377SEli Cohen for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { 22451a86b377SEli Cohen mvq = &ndev->vqs[i]; 22461a86b377SEli Cohen if (!mvq->initialized) 22471a86b377SEli Cohen continue; 22481a86b377SEli Cohen 22491a86b377SEli Cohen teardown_vq(ndev, mvq); 22501a86b377SEli Cohen } 22511a86b377SEli Cohen } 22521a86b377SEli Cohen 2253e4fc6650SEli Cohen static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) 22548a7c3213SMichael S. Tsirkin { 2255e4fc6650SEli Cohen if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_CTRL_VQ)) { 2256e4fc6650SEli Cohen if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) { 2257e4fc6650SEli Cohen /* MQ supported. CVQ index is right above the last data virtqueue's */ 2258e4fc6650SEli Cohen mvdev->max_idx = mvdev->max_vqs; 2259e4fc6650SEli Cohen } else { 2260e4fc6650SEli Cohen /* Only CVQ supportted. data virtqueues occupy indices 0 and 1. 2261e4fc6650SEli Cohen * CVQ gets index 2 2262e4fc6650SEli Cohen */ 2263e4fc6650SEli Cohen mvdev->max_idx = 2; 22648a7c3213SMichael S. Tsirkin } 2265e4fc6650SEli Cohen } else { 2266e4fc6650SEli Cohen /* Two data virtqueues only: one for rx and one for tx */ 2267e4fc6650SEli Cohen mvdev->max_idx = 1; 2268e4fc6650SEli Cohen } 226936bdcf31SEli Cohen } 227036bdcf31SEli Cohen 2271a64917bcSEli Cohen static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) 22721a86b377SEli Cohen { 22731a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22741a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 22751a86b377SEli Cohen int err; 22761a86b377SEli Cohen 22771a86b377SEli Cohen print_features(mvdev, features, true); 22781a86b377SEli Cohen 227930c22f38SSi-Wei Liu err = verify_driver_features(mvdev, features); 22801a86b377SEli Cohen if (err) 22811a86b377SEli Cohen return err; 22821a86b377SEli Cohen 22831a86b377SEli Cohen ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; 2284b03fc43eSEli Cohen if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)) 2285acde3929SEli Cohen ndev->rqt_size = mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs); 2286b03fc43eSEli Cohen else 2287acde3929SEli Cohen ndev->rqt_size = 1; 2288acde3929SEli Cohen 2289acde3929SEli Cohen ndev->cur_num_vqs = 2 * ndev->rqt_size; 2290b03fc43eSEli Cohen 2291e4fc6650SEli Cohen update_cvq_info(mvdev); 22921a86b377SEli Cohen return err; 22931a86b377SEli Cohen } 22941a86b377SEli Cohen 22951a86b377SEli Cohen static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 22961a86b377SEli Cohen { 2297edf747afSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2298edf747afSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2299edf747afSEli Cohen 2300edf747afSEli Cohen ndev->config_cb = *cb; 23011a86b377SEli Cohen } 23021a86b377SEli Cohen 23031a86b377SEli Cohen #define MLX5_VDPA_MAX_VQ_ENTRIES 256 23041a86b377SEli Cohen static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev) 23051a86b377SEli Cohen { 23061a86b377SEli Cohen return MLX5_VDPA_MAX_VQ_ENTRIES; 23071a86b377SEli Cohen } 23081a86b377SEli Cohen 23091a86b377SEli Cohen static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev) 23101a86b377SEli Cohen { 23111a86b377SEli Cohen return VIRTIO_ID_NET; 23121a86b377SEli Cohen } 23131a86b377SEli Cohen 23141a86b377SEli Cohen static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev) 23151a86b377SEli Cohen { 23161a86b377SEli Cohen return PCI_VENDOR_ID_MELLANOX; 23171a86b377SEli Cohen } 23181a86b377SEli Cohen 23191a86b377SEli Cohen static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev) 23201a86b377SEli Cohen { 23211a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 23221a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 23231a86b377SEli Cohen 23241a86b377SEli Cohen print_status(mvdev, ndev->mvdev.status, false); 23251a86b377SEli Cohen return ndev->mvdev.status; 23261a86b377SEli Cohen } 23271a86b377SEli Cohen 23281a86b377SEli Cohen static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 23291a86b377SEli Cohen { 23301a86b377SEli Cohen struct mlx5_vq_restore_info *ri = &mvq->ri; 233152893733SEli Cohen struct mlx5_virtq_attr attr = {}; 23321a86b377SEli Cohen int err; 23331a86b377SEli Cohen 233452893733SEli Cohen if (mvq->initialized) { 23351a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 23361a86b377SEli Cohen if (err) 23371a86b377SEli Cohen return err; 233852893733SEli Cohen } 23391a86b377SEli Cohen 23401a86b377SEli Cohen ri->avail_index = attr.available_index; 2341b35ccebeSEli Cohen ri->used_index = attr.used_index; 23421a86b377SEli Cohen ri->ready = mvq->ready; 23431a86b377SEli Cohen ri->num_ent = mvq->num_ent; 23441a86b377SEli Cohen ri->desc_addr = mvq->desc_addr; 23451a86b377SEli Cohen ri->device_addr = mvq->device_addr; 23461a86b377SEli Cohen ri->driver_addr = mvq->driver_addr; 23471a86b377SEli Cohen ri->restore = true; 23481a86b377SEli Cohen return 0; 23491a86b377SEli Cohen } 23501a86b377SEli Cohen 23511a86b377SEli Cohen static int save_channels_info(struct mlx5_vdpa_net *ndev) 23521a86b377SEli Cohen { 23531a86b377SEli Cohen int i; 23541a86b377SEli Cohen 23551a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 23561a86b377SEli Cohen memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); 23571a86b377SEli Cohen save_channel_info(ndev, &ndev->vqs[i]); 23581a86b377SEli Cohen } 23591a86b377SEli Cohen return 0; 23601a86b377SEli Cohen } 23611a86b377SEli Cohen 23621a86b377SEli Cohen static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) 23631a86b377SEli Cohen { 23641a86b377SEli Cohen int i; 23651a86b377SEli Cohen 23661a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 23671a86b377SEli Cohen memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 23681a86b377SEli Cohen } 23691a86b377SEli Cohen 23701a86b377SEli Cohen static void restore_channels_info(struct mlx5_vdpa_net *ndev) 23711a86b377SEli Cohen { 23721a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 23731a86b377SEli Cohen struct mlx5_vq_restore_info *ri; 23741a86b377SEli Cohen int i; 23751a86b377SEli Cohen 23761a86b377SEli Cohen mlx5_clear_vqs(ndev); 23771a86b377SEli Cohen init_mvqs(ndev); 23781a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 23791a86b377SEli Cohen mvq = &ndev->vqs[i]; 23801a86b377SEli Cohen ri = &mvq->ri; 23811a86b377SEli Cohen if (!ri->restore) 23821a86b377SEli Cohen continue; 23831a86b377SEli Cohen 23841a86b377SEli Cohen mvq->avail_idx = ri->avail_index; 2385b35ccebeSEli Cohen mvq->used_idx = ri->used_index; 23861a86b377SEli Cohen mvq->ready = ri->ready; 23871a86b377SEli Cohen mvq->num_ent = ri->num_ent; 23881a86b377SEli Cohen mvq->desc_addr = ri->desc_addr; 23891a86b377SEli Cohen mvq->device_addr = ri->device_addr; 23901a86b377SEli Cohen mvq->driver_addr = ri->driver_addr; 23911a86b377SEli Cohen } 23921a86b377SEli Cohen } 23931a86b377SEli Cohen 2394ae0428deSEli Cohen static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) 23951a86b377SEli Cohen { 2396ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 23971a86b377SEli Cohen int err; 23981a86b377SEli Cohen 23991a86b377SEli Cohen suspend_vqs(ndev); 24001a86b377SEli Cohen err = save_channels_info(ndev); 24011a86b377SEli Cohen if (err) 24021a86b377SEli Cohen goto err_mr; 24031a86b377SEli Cohen 24041a86b377SEli Cohen teardown_driver(ndev); 2405ae0428deSEli Cohen mlx5_vdpa_destroy_mr(mvdev); 2406ae0428deSEli Cohen err = mlx5_vdpa_create_mr(mvdev, iotlb); 24071a86b377SEli Cohen if (err) 24081a86b377SEli Cohen goto err_mr; 24091a86b377SEli Cohen 2410ae0428deSEli Cohen if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 24111c80cf03SJason Wang goto err_mr; 24121897f0b6SEli Cohen 24131a86b377SEli Cohen restore_channels_info(ndev); 2414ae0428deSEli Cohen err = setup_driver(mvdev); 24151a86b377SEli Cohen if (err) 24161a86b377SEli Cohen goto err_setup; 24171a86b377SEli Cohen 24181a86b377SEli Cohen return 0; 24191a86b377SEli Cohen 24201a86b377SEli Cohen err_setup: 2421ae0428deSEli Cohen mlx5_vdpa_destroy_mr(mvdev); 24221a86b377SEli Cohen err_mr: 24231a86b377SEli Cohen return err; 24241a86b377SEli Cohen } 24251a86b377SEli Cohen 24261c80cf03SJason Wang /* reslock must be held for this function */ 2427ae0428deSEli Cohen static int setup_driver(struct mlx5_vdpa_dev *mvdev) 24281a86b377SEli Cohen { 2429ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 24301a86b377SEli Cohen int err; 24311a86b377SEli Cohen 2432759ae7f9SEli Cohen WARN_ON(!rwsem_is_locked(&ndev->reslock)); 24331c80cf03SJason Wang 24341a86b377SEli Cohen if (ndev->setup) { 2435ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n"); 24361a86b377SEli Cohen err = 0; 24371a86b377SEli Cohen goto out; 24381a86b377SEli Cohen } 2439ae0428deSEli Cohen err = setup_virtqueues(mvdev); 24401a86b377SEli Cohen if (err) { 2441ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "setup_virtqueues\n"); 24421a86b377SEli Cohen goto out; 24431a86b377SEli Cohen } 24441a86b377SEli Cohen 24451a86b377SEli Cohen err = create_rqt(ndev); 24461a86b377SEli Cohen if (err) { 2447ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "create_rqt\n"); 24481a86b377SEli Cohen goto err_rqt; 24491a86b377SEli Cohen } 24501a86b377SEli Cohen 24511a86b377SEli Cohen err = create_tir(ndev); 24521a86b377SEli Cohen if (err) { 2453ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "create_tir\n"); 24541a86b377SEli Cohen goto err_tir; 24551a86b377SEli Cohen } 24561a86b377SEli Cohen 2457baf2ad3fSEli Cohen err = setup_steering(ndev); 24581a86b377SEli Cohen if (err) { 2459baf2ad3fSEli Cohen mlx5_vdpa_warn(mvdev, "setup_steering\n"); 24601a86b377SEli Cohen goto err_fwd; 24611a86b377SEli Cohen } 24621a86b377SEli Cohen ndev->setup = true; 24631a86b377SEli Cohen 24641a86b377SEli Cohen return 0; 24651a86b377SEli Cohen 24661a86b377SEli Cohen err_fwd: 24671a86b377SEli Cohen destroy_tir(ndev); 24681a86b377SEli Cohen err_tir: 24691a86b377SEli Cohen destroy_rqt(ndev); 24701a86b377SEli Cohen err_rqt: 24711a86b377SEli Cohen teardown_virtqueues(ndev); 24721a86b377SEli Cohen out: 24731a86b377SEli Cohen return err; 24741a86b377SEli Cohen } 24751a86b377SEli Cohen 24761c80cf03SJason Wang /* reslock must be held for this function */ 24771a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev) 24781a86b377SEli Cohen { 24791c80cf03SJason Wang 2480759ae7f9SEli Cohen WARN_ON(!rwsem_is_locked(&ndev->reslock)); 24811c80cf03SJason Wang 24821a86b377SEli Cohen if (!ndev->setup) 24831c80cf03SJason Wang return; 24841a86b377SEli Cohen 2485baf2ad3fSEli Cohen teardown_steering(ndev); 24861a86b377SEli Cohen destroy_tir(ndev); 24871a86b377SEli Cohen destroy_rqt(ndev); 24881a86b377SEli Cohen teardown_virtqueues(ndev); 24891a86b377SEli Cohen ndev->setup = false; 24901a86b377SEli Cohen } 24911a86b377SEli Cohen 2492e3aadf2eSEli Cohen static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) 2493e3aadf2eSEli Cohen { 2494e3aadf2eSEli Cohen int i; 2495e3aadf2eSEli Cohen 2496e3aadf2eSEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 2497e3aadf2eSEli Cohen ndev->vqs[i].ready = false; 2498ef12e4bfSEli Cohen 2499ef12e4bfSEli Cohen ndev->mvdev.cvq.ready = false; 2500e3aadf2eSEli Cohen } 2501e3aadf2eSEli Cohen 2502ace92524SEli Cohen static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev) 2503ace92524SEli Cohen { 2504ace92524SEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 2505ace92524SEli Cohen int err = 0; 2506ace92524SEli Cohen 2507ace92524SEli Cohen if (mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) 2508ace92524SEli Cohen err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features, 2509ace92524SEli Cohen MLX5_CVQ_MAX_ENT, false, 2510ace92524SEli Cohen (struct vring_desc *)(uintptr_t)cvq->desc_addr, 2511ace92524SEli Cohen (struct vring_avail *)(uintptr_t)cvq->driver_addr, 2512ace92524SEli Cohen (struct vring_used *)(uintptr_t)cvq->device_addr); 2513ace92524SEli Cohen 2514ace92524SEli Cohen return err; 2515ace92524SEli Cohen } 2516ace92524SEli Cohen 25171a86b377SEli Cohen static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) 25181a86b377SEli Cohen { 25191a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 25201a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 25211a86b377SEli Cohen int err; 25221a86b377SEli Cohen 25231a86b377SEli Cohen print_status(mvdev, status, true); 25241a86b377SEli Cohen 2525759ae7f9SEli Cohen down_write(&ndev->reslock); 25261c80cf03SJason Wang 25271a86b377SEli Cohen if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { 25281a86b377SEli Cohen if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 2529ace92524SEli Cohen err = setup_cvq_vring(mvdev); 2530ace92524SEli Cohen if (err) { 2531ace92524SEli Cohen mlx5_vdpa_warn(mvdev, "failed to setup control VQ vring\n"); 2532ace92524SEli Cohen goto err_setup; 2533ace92524SEli Cohen } 2534ae0428deSEli Cohen err = setup_driver(mvdev); 25351a86b377SEli Cohen if (err) { 25361a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); 25371a86b377SEli Cohen goto err_setup; 25381a86b377SEli Cohen } 25391a86b377SEli Cohen } else { 25401a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); 25411c80cf03SJason Wang goto err_clear; 25421a86b377SEli Cohen } 25431a86b377SEli Cohen } 25441a86b377SEli Cohen 25451a86b377SEli Cohen ndev->mvdev.status = status; 2546759ae7f9SEli Cohen up_write(&ndev->reslock); 25471a86b377SEli Cohen return; 25481a86b377SEli Cohen 25491a86b377SEli Cohen err_setup: 25501a86b377SEli Cohen mlx5_vdpa_destroy_mr(&ndev->mvdev); 25511a86b377SEli Cohen ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; 25521c80cf03SJason Wang err_clear: 2553759ae7f9SEli Cohen up_write(&ndev->reslock); 25541a86b377SEli Cohen } 25551a86b377SEli Cohen 25568fcd20c3SEli Cohen static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev) 25578fcd20c3SEli Cohen { 25588fcd20c3SEli Cohen int i; 25598fcd20c3SEli Cohen 25608fcd20c3SEli Cohen /* default mapping all groups are mapped to asid 0 */ 25618fcd20c3SEli Cohen for (i = 0; i < MLX5_VDPA_NUMVQ_GROUPS; i++) 25628fcd20c3SEli Cohen mvdev->group2asid[i] = 0; 25638fcd20c3SEli Cohen } 25648fcd20c3SEli Cohen 25650686082dSXie Yongji static int mlx5_vdpa_reset(struct vdpa_device *vdev) 25660686082dSXie Yongji { 25670686082dSXie Yongji struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 25680686082dSXie Yongji struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 25690686082dSXie Yongji 25700686082dSXie Yongji print_status(mvdev, 0, true); 25710686082dSXie Yongji mlx5_vdpa_info(mvdev, "performing device reset\n"); 25721c80cf03SJason Wang 2573759ae7f9SEli Cohen down_write(&ndev->reslock); 25740686082dSXie Yongji teardown_driver(ndev); 25750686082dSXie Yongji clear_vqs_ready(ndev); 25760686082dSXie Yongji mlx5_vdpa_destroy_mr(&ndev->mvdev); 25770686082dSXie Yongji ndev->mvdev.status = 0; 2578b03fc43eSEli Cohen ndev->cur_num_vqs = 0; 25791892a3d4SEli Cohen ndev->mvdev.cvq.received_desc = 0; 25801892a3d4SEli Cohen ndev->mvdev.cvq.completed_desc = 0; 258175560522SEli Cohen memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1)); 25820686082dSXie Yongji ndev->mvdev.actual_features = 0; 25838fcd20c3SEli Cohen init_group_to_asid_map(mvdev); 25840686082dSXie Yongji ++mvdev->generation; 25858fcd20c3SEli Cohen 25860686082dSXie Yongji if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 25870686082dSXie Yongji if (mlx5_vdpa_create_mr(mvdev, NULL)) 25880686082dSXie Yongji mlx5_vdpa_warn(mvdev, "create MR failed\n"); 25890686082dSXie Yongji } 2590759ae7f9SEli Cohen up_write(&ndev->reslock); 25910686082dSXie Yongji 25920686082dSXie Yongji return 0; 25930686082dSXie Yongji } 25940686082dSXie Yongji 2595442706f9SStefano Garzarella static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev) 2596442706f9SStefano Garzarella { 2597442706f9SStefano Garzarella return sizeof(struct virtio_net_config); 2598442706f9SStefano Garzarella } 2599442706f9SStefano Garzarella 26001a86b377SEli Cohen static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, 26011a86b377SEli Cohen unsigned int len) 26021a86b377SEli Cohen { 26031a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 26041a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 26051a86b377SEli Cohen 2606dcfde163SStefano Garzarella if (offset + len <= sizeof(struct virtio_net_config)) 26072874211fSDan Carpenter memcpy(buf, (u8 *)&ndev->config + offset, len); 26081a86b377SEli Cohen } 26091a86b377SEli Cohen 26101a86b377SEli Cohen static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf, 26111a86b377SEli Cohen unsigned int len) 26121a86b377SEli Cohen { 26131a86b377SEli Cohen /* not supported */ 26141a86b377SEli Cohen } 26151a86b377SEli Cohen 26161a86b377SEli Cohen static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) 26171a86b377SEli Cohen { 26181a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 26191a86b377SEli Cohen 26201a86b377SEli Cohen return mvdev->generation; 26211a86b377SEli Cohen } 26221a86b377SEli Cohen 26238fcd20c3SEli Cohen static int set_map_control(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) 26241a86b377SEli Cohen { 26258fcd20c3SEli Cohen u64 start = 0ULL, last = 0ULL - 1; 26268fcd20c3SEli Cohen struct vhost_iotlb_map *map; 26278fcd20c3SEli Cohen int err = 0; 26288fcd20c3SEli Cohen 26298fcd20c3SEli Cohen spin_lock(&mvdev->cvq.iommu_lock); 26308fcd20c3SEli Cohen vhost_iotlb_reset(mvdev->cvq.iotlb); 26318fcd20c3SEli Cohen 26328fcd20c3SEli Cohen for (map = vhost_iotlb_itree_first(iotlb, start, last); map; 26338fcd20c3SEli Cohen map = vhost_iotlb_itree_next(map, start, last)) { 26348fcd20c3SEli Cohen err = vhost_iotlb_add_range(mvdev->cvq.iotlb, map->start, 26358fcd20c3SEli Cohen map->last, map->addr, map->perm); 26368fcd20c3SEli Cohen if (err) 26378fcd20c3SEli Cohen goto out; 26388fcd20c3SEli Cohen } 26398fcd20c3SEli Cohen 26408fcd20c3SEli Cohen out: 26418fcd20c3SEli Cohen spin_unlock(&mvdev->cvq.iommu_lock); 26428fcd20c3SEli Cohen return err; 26438fcd20c3SEli Cohen } 26448fcd20c3SEli Cohen 26458fcd20c3SEli Cohen static int set_map_data(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) 26468fcd20c3SEli Cohen { 26471a86b377SEli Cohen bool change_map; 26481a86b377SEli Cohen int err; 26491a86b377SEli Cohen 26501a86b377SEli Cohen err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map); 26511a86b377SEli Cohen if (err) { 26521a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); 26538fcd20c3SEli Cohen return err; 26541a86b377SEli Cohen } 26551a86b377SEli Cohen 26561a86b377SEli Cohen if (change_map) 26571c80cf03SJason Wang err = mlx5_vdpa_change_map(mvdev, iotlb); 26581a86b377SEli Cohen 26598fcd20c3SEli Cohen return err; 26608fcd20c3SEli Cohen } 26618fcd20c3SEli Cohen 26628fcd20c3SEli Cohen static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid, 26638fcd20c3SEli Cohen struct vhost_iotlb *iotlb) 26648fcd20c3SEli Cohen { 26658fcd20c3SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 26668fcd20c3SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 266793e530d2SEli Cohen int err = -EINVAL; 26688fcd20c3SEli Cohen 26698fcd20c3SEli Cohen down_write(&ndev->reslock); 26708fcd20c3SEli Cohen if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] == asid) { 26718fcd20c3SEli Cohen err = set_map_data(mvdev, iotlb); 26728fcd20c3SEli Cohen if (err) 26738fcd20c3SEli Cohen goto out; 26748fcd20c3SEli Cohen } 26758fcd20c3SEli Cohen 26768fcd20c3SEli Cohen if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] == asid) 26778fcd20c3SEli Cohen err = set_map_control(mvdev, iotlb); 26788fcd20c3SEli Cohen 26798fcd20c3SEli Cohen out: 2680759ae7f9SEli Cohen up_write(&ndev->reslock); 26811c80cf03SJason Wang return err; 26821a86b377SEli Cohen } 26831a86b377SEli Cohen 26841a86b377SEli Cohen static void mlx5_vdpa_free(struct vdpa_device *vdev) 26851a86b377SEli Cohen { 26861a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 26877c9f131fSEli Cohen struct mlx5_core_dev *pfmdev; 26881a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 26891a86b377SEli Cohen 26901a86b377SEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 26911a86b377SEli Cohen 26921a86b377SEli Cohen free_resources(ndev); 26936f5312f8SEli Cohen mlx5_vdpa_destroy_mr(mvdev); 26947c9f131fSEli Cohen if (!is_zero_ether_addr(ndev->config.mac)) { 26957c9f131fSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 26967c9f131fSEli Cohen mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); 26977c9f131fSEli Cohen } 26981a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 269975560522SEli Cohen kfree(ndev->event_cbs); 270075560522SEli Cohen kfree(ndev->vqs); 27011a86b377SEli Cohen } 27021a86b377SEli Cohen 27031a86b377SEli Cohen static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx) 27041a86b377SEli Cohen { 2705b57c46cbSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 27061a86b377SEli Cohen struct vdpa_notification_area ret = {}; 2707b57c46cbSEli Cohen struct mlx5_vdpa_net *ndev; 2708b57c46cbSEli Cohen phys_addr_t addr; 27091a86b377SEli Cohen 27105262912eSEli Cohen if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2711e4fc6650SEli Cohen return ret; 2712e4fc6650SEli Cohen 2713b57c46cbSEli Cohen /* If SF BAR size is smaller than PAGE_SIZE, do not use direct 2714b57c46cbSEli Cohen * notification to avoid the risk of mapping pages that contain BAR of more 2715b57c46cbSEli Cohen * than one SF 2716b57c46cbSEli Cohen */ 2717b57c46cbSEli Cohen if (MLX5_CAP_GEN(mvdev->mdev, log_min_sf_size) + 12 < PAGE_SHIFT) 2718b57c46cbSEli Cohen return ret; 2719b57c46cbSEli Cohen 2720b57c46cbSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 2721b57c46cbSEli Cohen addr = (phys_addr_t)ndev->mvdev.res.phys_kick_addr; 2722b57c46cbSEli Cohen ret.addr = addr; 2723b57c46cbSEli Cohen ret.size = PAGE_SIZE; 27241a86b377SEli Cohen return ret; 27251a86b377SEli Cohen } 27261a86b377SEli Cohen 27271a86b377SEli Cohen static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) 27281a86b377SEli Cohen { 27291a86b377SEli Cohen return -EOPNOTSUPP; 27301a86b377SEli Cohen } 27311a86b377SEli Cohen 2732a64917bcSEli Cohen static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) 2733a64917bcSEli Cohen { 2734a64917bcSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2735a64917bcSEli Cohen 2736a64917bcSEli Cohen return mvdev->actual_features; 2737a64917bcSEli Cohen } 2738a64917bcSEli Cohen 27391892a3d4SEli Cohen static int counter_set_query(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 27401892a3d4SEli Cohen u64 *received_desc, u64 *completed_desc) 27411892a3d4SEli Cohen { 27421892a3d4SEli Cohen u32 in[MLX5_ST_SZ_DW(query_virtio_q_counters_in)] = {}; 27431892a3d4SEli Cohen u32 out[MLX5_ST_SZ_DW(query_virtio_q_counters_out)] = {}; 27441892a3d4SEli Cohen void *cmd_hdr; 27451892a3d4SEli Cohen void *ctx; 27461892a3d4SEli Cohen int err; 27471892a3d4SEli Cohen 27481892a3d4SEli Cohen if (!counters_supported(&ndev->mvdev)) 27491892a3d4SEli Cohen return -EOPNOTSUPP; 27501892a3d4SEli Cohen 27511892a3d4SEli Cohen if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 27521892a3d4SEli Cohen return -EAGAIN; 27531892a3d4SEli Cohen 27541892a3d4SEli Cohen cmd_hdr = MLX5_ADDR_OF(query_virtio_q_counters_in, in, hdr); 27551892a3d4SEli Cohen 27561892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 27571892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 27581892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 27591892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->counter_set_id); 27601892a3d4SEli Cohen 27611892a3d4SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 27621892a3d4SEli Cohen if (err) 27631892a3d4SEli Cohen return err; 27641892a3d4SEli Cohen 27651892a3d4SEli Cohen ctx = MLX5_ADDR_OF(query_virtio_q_counters_out, out, counters); 27661892a3d4SEli Cohen *received_desc = MLX5_GET64(virtio_q_counters, ctx, received_desc); 27671892a3d4SEli Cohen *completed_desc = MLX5_GET64(virtio_q_counters, ctx, completed_desc); 27681892a3d4SEli Cohen return 0; 27691892a3d4SEli Cohen } 27701892a3d4SEli Cohen 27711892a3d4SEli Cohen static int mlx5_vdpa_get_vendor_vq_stats(struct vdpa_device *vdev, u16 idx, 27721892a3d4SEli Cohen struct sk_buff *msg, 27731892a3d4SEli Cohen struct netlink_ext_ack *extack) 27741892a3d4SEli Cohen { 27751892a3d4SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 27761892a3d4SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 27771892a3d4SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 27781892a3d4SEli Cohen struct mlx5_control_vq *cvq; 27791892a3d4SEli Cohen u64 received_desc; 27801892a3d4SEli Cohen u64 completed_desc; 27811892a3d4SEli Cohen int err = 0; 27821892a3d4SEli Cohen 2783759ae7f9SEli Cohen down_read(&ndev->reslock); 27841892a3d4SEli Cohen if (!is_index_valid(mvdev, idx)) { 27851892a3d4SEli Cohen NL_SET_ERR_MSG_MOD(extack, "virtqueue index is not valid"); 27861892a3d4SEli Cohen err = -EINVAL; 27871892a3d4SEli Cohen goto out_err; 27881892a3d4SEli Cohen } 27891892a3d4SEli Cohen 27901892a3d4SEli Cohen if (idx == ctrl_vq_idx(mvdev)) { 27911892a3d4SEli Cohen cvq = &mvdev->cvq; 27921892a3d4SEli Cohen received_desc = cvq->received_desc; 27931892a3d4SEli Cohen completed_desc = cvq->completed_desc; 27941892a3d4SEli Cohen goto out; 27951892a3d4SEli Cohen } 27961892a3d4SEli Cohen 27971892a3d4SEli Cohen mvq = &ndev->vqs[idx]; 27981892a3d4SEli Cohen err = counter_set_query(ndev, mvq, &received_desc, &completed_desc); 27991892a3d4SEli Cohen if (err) { 28001892a3d4SEli Cohen NL_SET_ERR_MSG_MOD(extack, "failed to query hardware"); 28011892a3d4SEli Cohen goto out_err; 28021892a3d4SEli Cohen } 28031892a3d4SEli Cohen 28041892a3d4SEli Cohen out: 28051892a3d4SEli Cohen err = -EMSGSIZE; 28061892a3d4SEli Cohen if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "received_desc")) 28071892a3d4SEli Cohen goto out_err; 28081892a3d4SEli Cohen 28091892a3d4SEli Cohen if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, received_desc, 28101892a3d4SEli Cohen VDPA_ATTR_PAD)) 28111892a3d4SEli Cohen goto out_err; 28121892a3d4SEli Cohen 28131892a3d4SEli Cohen if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "completed_desc")) 28141892a3d4SEli Cohen goto out_err; 28151892a3d4SEli Cohen 28161892a3d4SEli Cohen if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, completed_desc, 28171892a3d4SEli Cohen VDPA_ATTR_PAD)) 28181892a3d4SEli Cohen goto out_err; 28191892a3d4SEli Cohen 28201892a3d4SEli Cohen err = 0; 28211892a3d4SEli Cohen out_err: 2822759ae7f9SEli Cohen up_read(&ndev->reslock); 28231892a3d4SEli Cohen return err; 28241892a3d4SEli Cohen } 28251892a3d4SEli Cohen 2826cae15c2eSEli Cohen static void mlx5_vdpa_cvq_suspend(struct mlx5_vdpa_dev *mvdev) 2827cae15c2eSEli Cohen { 2828cae15c2eSEli Cohen struct mlx5_control_vq *cvq; 2829cae15c2eSEli Cohen 2830cae15c2eSEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 2831cae15c2eSEli Cohen return; 2832cae15c2eSEli Cohen 2833cae15c2eSEli Cohen cvq = &mvdev->cvq; 2834cae15c2eSEli Cohen cvq->ready = false; 2835cae15c2eSEli Cohen } 2836cae15c2eSEli Cohen 2837cae15c2eSEli Cohen static int mlx5_vdpa_suspend(struct vdpa_device *vdev) 2838cae15c2eSEli Cohen { 2839cae15c2eSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2840cae15c2eSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2841cae15c2eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 2842cae15c2eSEli Cohen int i; 2843cae15c2eSEli Cohen 2844cae15c2eSEli Cohen down_write(&ndev->reslock); 2845cae15c2eSEli Cohen mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 2846cae15c2eSEli Cohen ndev->nb_registered = false; 2847cae15c2eSEli Cohen flush_workqueue(ndev->mvdev.wq); 2848cae15c2eSEli Cohen for (i = 0; i < ndev->cur_num_vqs; i++) { 2849cae15c2eSEli Cohen mvq = &ndev->vqs[i]; 2850cae15c2eSEli Cohen suspend_vq(ndev, mvq); 2851cae15c2eSEli Cohen } 2852cae15c2eSEli Cohen mlx5_vdpa_cvq_suspend(mvdev); 2853cae15c2eSEli Cohen up_write(&ndev->reslock); 2854cae15c2eSEli Cohen return 0; 2855cae15c2eSEli Cohen } 2856cae15c2eSEli Cohen 28578fcd20c3SEli Cohen static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group, 28588fcd20c3SEli Cohen unsigned int asid) 28598fcd20c3SEli Cohen { 28608fcd20c3SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 28618fcd20c3SEli Cohen 28628fcd20c3SEli Cohen if (group >= MLX5_VDPA_NUMVQ_GROUPS) 28638fcd20c3SEli Cohen return -EINVAL; 28648fcd20c3SEli Cohen 28658fcd20c3SEli Cohen mvdev->group2asid[group] = asid; 28668fcd20c3SEli Cohen return 0; 28678fcd20c3SEli Cohen } 28688fcd20c3SEli Cohen 28691a86b377SEli Cohen static const struct vdpa_config_ops mlx5_vdpa_ops = { 28701a86b377SEli Cohen .set_vq_address = mlx5_vdpa_set_vq_address, 28711a86b377SEli Cohen .set_vq_num = mlx5_vdpa_set_vq_num, 28721a86b377SEli Cohen .kick_vq = mlx5_vdpa_kick_vq, 28731a86b377SEli Cohen .set_vq_cb = mlx5_vdpa_set_vq_cb, 28741a86b377SEli Cohen .set_vq_ready = mlx5_vdpa_set_vq_ready, 28751a86b377SEli Cohen .get_vq_ready = mlx5_vdpa_get_vq_ready, 28761a86b377SEli Cohen .set_vq_state = mlx5_vdpa_set_vq_state, 28771a86b377SEli Cohen .get_vq_state = mlx5_vdpa_get_vq_state, 28781892a3d4SEli Cohen .get_vendor_vq_stats = mlx5_vdpa_get_vendor_vq_stats, 28791a86b377SEli Cohen .get_vq_notification = mlx5_get_vq_notification, 28801a86b377SEli Cohen .get_vq_irq = mlx5_get_vq_irq, 28811a86b377SEli Cohen .get_vq_align = mlx5_vdpa_get_vq_align, 2882d4821902SGautam Dawar .get_vq_group = mlx5_vdpa_get_vq_group, 2883a64917bcSEli Cohen .get_device_features = mlx5_vdpa_get_device_features, 2884a64917bcSEli Cohen .set_driver_features = mlx5_vdpa_set_driver_features, 2885a64917bcSEli Cohen .get_driver_features = mlx5_vdpa_get_driver_features, 28861a86b377SEli Cohen .set_config_cb = mlx5_vdpa_set_config_cb, 28871a86b377SEli Cohen .get_vq_num_max = mlx5_vdpa_get_vq_num_max, 28881a86b377SEli Cohen .get_device_id = mlx5_vdpa_get_device_id, 28891a86b377SEli Cohen .get_vendor_id = mlx5_vdpa_get_vendor_id, 28901a86b377SEli Cohen .get_status = mlx5_vdpa_get_status, 28911a86b377SEli Cohen .set_status = mlx5_vdpa_set_status, 28920686082dSXie Yongji .reset = mlx5_vdpa_reset, 2893442706f9SStefano Garzarella .get_config_size = mlx5_vdpa_get_config_size, 28941a86b377SEli Cohen .get_config = mlx5_vdpa_get_config, 28951a86b377SEli Cohen .set_config = mlx5_vdpa_set_config, 28961a86b377SEli Cohen .get_generation = mlx5_vdpa_get_generation, 28971a86b377SEli Cohen .set_map = mlx5_vdpa_set_map, 28988fcd20c3SEli Cohen .set_group_asid = mlx5_set_group_asid, 28991a86b377SEli Cohen .free = mlx5_vdpa_free, 2900cae15c2eSEli Cohen .suspend = mlx5_vdpa_suspend, 29011a86b377SEli Cohen }; 29021a86b377SEli Cohen 2903d084d996SSi-Wei Liu static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) 2904d084d996SSi-Wei Liu { 2905d084d996SSi-Wei Liu u16 hw_mtu; 2906d084d996SSi-Wei Liu int err; 2907d084d996SSi-Wei Liu 2908d084d996SSi-Wei Liu err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2909d084d996SSi-Wei Liu if (err) 2910d084d996SSi-Wei Liu return err; 2911d084d996SSi-Wei Liu 2912d084d996SSi-Wei Liu *mtu = hw_mtu - MLX5V_ETH_HARD_MTU; 2913d084d996SSi-Wei Liu return 0; 2914d084d996SSi-Wei Liu } 2915d084d996SSi-Wei Liu 29161a86b377SEli Cohen static int alloc_resources(struct mlx5_vdpa_net *ndev) 29171a86b377SEli Cohen { 29181a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 29191a86b377SEli Cohen int err; 29201a86b377SEli Cohen 29211a86b377SEli Cohen if (res->valid) { 29221a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); 29231a86b377SEli Cohen return -EEXIST; 29241a86b377SEli Cohen } 29251a86b377SEli Cohen 29261a86b377SEli Cohen err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); 29271a86b377SEli Cohen if (err) 29281a86b377SEli Cohen return err; 29291a86b377SEli Cohen 29301a86b377SEli Cohen err = create_tis(ndev); 29311a86b377SEli Cohen if (err) 29321a86b377SEli Cohen goto err_tis; 29331a86b377SEli Cohen 29341a86b377SEli Cohen res->valid = true; 29351a86b377SEli Cohen 29361a86b377SEli Cohen return 0; 29371a86b377SEli Cohen 29381a86b377SEli Cohen err_tis: 29391a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 29401a86b377SEli Cohen return err; 29411a86b377SEli Cohen } 29421a86b377SEli Cohen 29431a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev) 29441a86b377SEli Cohen { 29451a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 29461a86b377SEli Cohen 29471a86b377SEli Cohen if (!res->valid) 29481a86b377SEli Cohen return; 29491a86b377SEli Cohen 29501a86b377SEli Cohen destroy_tis(ndev); 29511a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 29521a86b377SEli Cohen res->valid = false; 29531a86b377SEli Cohen } 29541a86b377SEli Cohen 29551a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev) 29561a86b377SEli Cohen { 29571a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 29581a86b377SEli Cohen int i; 29591a86b377SEli Cohen 2960acde3929SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; ++i) { 29611a86b377SEli Cohen mvq = &ndev->vqs[i]; 29621a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 29631a86b377SEli Cohen mvq->index = i; 29641a86b377SEli Cohen mvq->ndev = ndev; 29651a86b377SEli Cohen mvq->fwqp.fw = true; 2966cae15c2eSEli Cohen mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; 29671a86b377SEli Cohen } 29681a86b377SEli Cohen for (; i < ndev->mvdev.max_vqs; i++) { 29691a86b377SEli Cohen mvq = &ndev->vqs[i]; 29701a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 29711a86b377SEli Cohen mvq->index = i; 29721a86b377SEli Cohen mvq->ndev = ndev; 29731a86b377SEli Cohen } 29741a86b377SEli Cohen } 29751a86b377SEli Cohen 297658926c8aSEli Cohen struct mlx5_vdpa_mgmtdev { 297758926c8aSEli Cohen struct vdpa_mgmt_dev mgtdev; 297858926c8aSEli Cohen struct mlx5_adev *madev; 297958926c8aSEli Cohen struct mlx5_vdpa_net *ndev; 298058926c8aSEli Cohen }; 298158926c8aSEli Cohen 2982edf747afSEli Cohen static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) 2983edf747afSEli Cohen { 2984edf747afSEli Cohen u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; 2985edf747afSEli Cohen u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; 2986edf747afSEli Cohen int err; 2987edf747afSEli Cohen 2988edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE); 2989edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, op_mod, opmod); 2990edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, vport_number, vport); 2991edf747afSEli Cohen if (vport) 2992edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, other_vport, 1); 2993edf747afSEli Cohen 2994edf747afSEli Cohen err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out); 2995edf747afSEli Cohen if (err) 2996edf747afSEli Cohen return 0; 2997edf747afSEli Cohen 2998edf747afSEli Cohen return MLX5_GET(query_vport_state_out, out, state); 2999edf747afSEli Cohen } 3000edf747afSEli Cohen 3001edf747afSEli Cohen static bool get_link_state(struct mlx5_vdpa_dev *mvdev) 3002edf747afSEli Cohen { 3003edf747afSEli Cohen if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) == 3004edf747afSEli Cohen VPORT_STATE_UP) 3005edf747afSEli Cohen return true; 3006edf747afSEli Cohen 3007edf747afSEli Cohen return false; 3008edf747afSEli Cohen } 3009edf747afSEli Cohen 3010edf747afSEli Cohen static void update_carrier(struct work_struct *work) 3011edf747afSEli Cohen { 3012edf747afSEli Cohen struct mlx5_vdpa_wq_ent *wqent; 3013edf747afSEli Cohen struct mlx5_vdpa_dev *mvdev; 3014edf747afSEli Cohen struct mlx5_vdpa_net *ndev; 3015edf747afSEli Cohen 3016edf747afSEli Cohen wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 3017edf747afSEli Cohen mvdev = wqent->mvdev; 3018edf747afSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 3019edf747afSEli Cohen if (get_link_state(mvdev)) 3020edf747afSEli Cohen ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 3021edf747afSEli Cohen else 3022edf747afSEli Cohen ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 3023edf747afSEli Cohen 3024edf747afSEli Cohen if (ndev->config_cb.callback) 3025edf747afSEli Cohen ndev->config_cb.callback(ndev->config_cb.private); 3026edf747afSEli Cohen 3027edf747afSEli Cohen kfree(wqent); 3028edf747afSEli Cohen } 3029edf747afSEli Cohen 3030edf747afSEli Cohen static int event_handler(struct notifier_block *nb, unsigned long event, void *param) 3031edf747afSEli Cohen { 3032edf747afSEli Cohen struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb); 3033edf747afSEli Cohen struct mlx5_eqe *eqe = param; 3034edf747afSEli Cohen int ret = NOTIFY_DONE; 3035edf747afSEli Cohen struct mlx5_vdpa_wq_ent *wqent; 3036edf747afSEli Cohen 3037edf747afSEli Cohen if (event == MLX5_EVENT_TYPE_PORT_CHANGE) { 3038edf747afSEli Cohen switch (eqe->sub_type) { 3039edf747afSEli Cohen case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 3040edf747afSEli Cohen case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 3041cae15c2eSEli Cohen down_read(&ndev->reslock); 3042cae15c2eSEli Cohen if (!ndev->nb_registered) { 3043cae15c2eSEli Cohen up_read(&ndev->reslock); 3044edf747afSEli Cohen return NOTIFY_DONE; 3045cae15c2eSEli Cohen } 3046cae15c2eSEli Cohen wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); 3047cae15c2eSEli Cohen if (!wqent) { 3048cae15c2eSEli Cohen up_read(&ndev->reslock); 3049cae15c2eSEli Cohen return NOTIFY_DONE; 3050cae15c2eSEli Cohen } 3051edf747afSEli Cohen 3052edf747afSEli Cohen wqent->mvdev = &ndev->mvdev; 3053edf747afSEli Cohen INIT_WORK(&wqent->work, update_carrier); 3054edf747afSEli Cohen queue_work(ndev->mvdev.wq, &wqent->work); 3055cae15c2eSEli Cohen up_read(&ndev->reslock); 3056edf747afSEli Cohen ret = NOTIFY_OK; 3057edf747afSEli Cohen break; 3058edf747afSEli Cohen default: 3059edf747afSEli Cohen return NOTIFY_DONE; 3060edf747afSEli Cohen } 3061edf747afSEli Cohen return ret; 3062edf747afSEli Cohen } 3063edf747afSEli Cohen return ret; 3064edf747afSEli Cohen } 3065edf747afSEli Cohen 30661e00e821SEli Cohen static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu) 30671e00e821SEli Cohen { 30681e00e821SEli Cohen int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); 30691e00e821SEli Cohen void *in; 30701e00e821SEli Cohen int err; 30711e00e821SEli Cohen 30721e00e821SEli Cohen in = kvzalloc(inlen, GFP_KERNEL); 30731e00e821SEli Cohen if (!in) 30741e00e821SEli Cohen return -ENOMEM; 30751e00e821SEli Cohen 30761e00e821SEli Cohen MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1); 30771e00e821SEli Cohen MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu, 30781e00e821SEli Cohen mtu + MLX5V_ETH_HARD_MTU); 30791e00e821SEli Cohen MLX5_SET(modify_nic_vport_context_in, in, opcode, 30801e00e821SEli Cohen MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); 30811e00e821SEli Cohen 30821e00e821SEli Cohen err = mlx5_cmd_exec_in(mdev, modify_nic_vport_context, in); 30831e00e821SEli Cohen 30841e00e821SEli Cohen kvfree(in); 30851e00e821SEli Cohen return err; 30861e00e821SEli Cohen } 30871e00e821SEli Cohen 3088d8ca2fa5SParav Pandit static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, 3089d8ca2fa5SParav Pandit const struct vdpa_dev_set_config *add_config) 30901a86b377SEli Cohen { 309158926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 30921a86b377SEli Cohen struct virtio_net_config *config; 30937c9f131fSEli Cohen struct mlx5_core_dev *pfmdev; 30941a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev; 30951a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 309658926c8aSEli Cohen struct mlx5_core_dev *mdev; 30971a86b377SEli Cohen u32 max_vqs; 3098246fd1caSEli Cohen u16 mtu; 30991a86b377SEli Cohen int err; 31001a86b377SEli Cohen 310158926c8aSEli Cohen if (mgtdev->ndev) 310258926c8aSEli Cohen return -ENOSPC; 310358926c8aSEli Cohen 310458926c8aSEli Cohen mdev = mgtdev->madev->mdev; 3105879753c8SEli Cohen if (!(MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_queue_type) & 3106879753c8SEli Cohen MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)) { 3107879753c8SEli Cohen dev_warn(mdev->device, "missing support for split virtqueues\n"); 3108879753c8SEli Cohen return -EOPNOTSUPP; 3109879753c8SEli Cohen } 3110879753c8SEli Cohen 3111acde3929SEli Cohen max_vqs = min_t(int, MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues), 3112acde3929SEli Cohen 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size)); 311375560522SEli Cohen if (max_vqs < 2) { 311475560522SEli Cohen dev_warn(mdev->device, 311575560522SEli Cohen "%d virtqueues are supported. At least 2 are required\n", 311675560522SEli Cohen max_vqs); 311775560522SEli Cohen return -EAGAIN; 311875560522SEli Cohen } 311975560522SEli Cohen 312075560522SEli Cohen if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP)) { 312175560522SEli Cohen if (add_config->net.max_vq_pairs > max_vqs / 2) 312275560522SEli Cohen return -EINVAL; 312375560522SEli Cohen max_vqs = min_t(u32, max_vqs, 2 * add_config->net.max_vq_pairs); 312475560522SEli Cohen } else { 312575560522SEli Cohen max_vqs = 2; 312675560522SEli Cohen } 31271a86b377SEli Cohen 31281a86b377SEli Cohen ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, 31298fcd20c3SEli Cohen MLX5_VDPA_NUMVQ_GROUPS, MLX5_VDPA_NUM_AS, name, false); 31301a86b377SEli Cohen if (IS_ERR(ndev)) 313174c9729dSLeon Romanovsky return PTR_ERR(ndev); 31321a86b377SEli Cohen 313379de65edSEli Cohen ndev->mvdev.mlx_features = mgtdev->mgtdev.supported_features; 31341a86b377SEli Cohen ndev->mvdev.max_vqs = max_vqs; 31351a86b377SEli Cohen mvdev = &ndev->mvdev; 31361a86b377SEli Cohen mvdev->mdev = mdev; 313775560522SEli Cohen 313875560522SEli Cohen ndev->vqs = kcalloc(max_vqs, sizeof(*ndev->vqs), GFP_KERNEL); 313975560522SEli Cohen ndev->event_cbs = kcalloc(max_vqs + 1, sizeof(*ndev->event_cbs), GFP_KERNEL); 314075560522SEli Cohen if (!ndev->vqs || !ndev->event_cbs) { 314175560522SEli Cohen err = -ENOMEM; 314275560522SEli Cohen goto err_alloc; 314375560522SEli Cohen } 314475560522SEli Cohen 31451a86b377SEli Cohen init_mvqs(ndev); 3146759ae7f9SEli Cohen init_rwsem(&ndev->reslock); 31471a86b377SEli Cohen config = &ndev->config; 31481e00e821SEli Cohen 31491e00e821SEli Cohen if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU)) { 31501e00e821SEli Cohen err = config_func_mtu(mdev, add_config->net.mtu); 31511e00e821SEli Cohen if (err) 3152759ae7f9SEli Cohen goto err_alloc; 31531e00e821SEli Cohen } 31541e00e821SEli Cohen 3155246fd1caSEli Cohen err = query_mtu(mdev, &mtu); 31561a86b377SEli Cohen if (err) 3157759ae7f9SEli Cohen goto err_alloc; 31581a86b377SEli Cohen 3159246fd1caSEli Cohen ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, mtu); 31601a86b377SEli Cohen 3161edf747afSEli Cohen if (get_link_state(mvdev)) 3162edf747afSEli Cohen ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 3163edf747afSEli Cohen else 3164edf747afSEli Cohen ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 3165edf747afSEli Cohen 3166a007d940SEli Cohen if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) { 3167a007d940SEli Cohen memcpy(ndev->config.mac, add_config->net.mac, ETH_ALEN); 3168a007d940SEli Cohen } else { 31691a86b377SEli Cohen err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac); 31701a86b377SEli Cohen if (err) 3171759ae7f9SEli Cohen goto err_alloc; 3172a007d940SEli Cohen } 31731a86b377SEli Cohen 31747c9f131fSEli Cohen if (!is_zero_ether_addr(config->mac)) { 31757c9f131fSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev)); 31767c9f131fSEli Cohen err = mlx5_mpfs_add_mac(pfmdev, config->mac); 31777c9f131fSEli Cohen if (err) 3178759ae7f9SEli Cohen goto err_alloc; 31795262912eSEli Cohen 31805262912eSEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MAC); 31817c9f131fSEli Cohen } 31827c9f131fSEli Cohen 3183acde3929SEli Cohen config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, max_vqs / 2); 31847d23dcdfSEli Cohen mvdev->vdev.dma_dev = &mdev->pdev->dev; 31851a86b377SEli Cohen err = mlx5_vdpa_alloc_resources(&ndev->mvdev); 31861a86b377SEli Cohen if (err) 31877c9f131fSEli Cohen goto err_mpfs; 31881a86b377SEli Cohen 31896f5312f8SEli Cohen if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 31906f5312f8SEli Cohen err = mlx5_vdpa_create_mr(mvdev, NULL); 31911a86b377SEli Cohen if (err) 31921a86b377SEli Cohen goto err_res; 31936f5312f8SEli Cohen } 31946f5312f8SEli Cohen 31956f5312f8SEli Cohen err = alloc_resources(ndev); 31966f5312f8SEli Cohen if (err) 31976f5312f8SEli Cohen goto err_mr; 31981a86b377SEli Cohen 319955ebf0d6SJason Wang ndev->cvq_ent.mvdev = mvdev; 320055ebf0d6SJason Wang INIT_WORK(&ndev->cvq_ent.work, mlx5_cvq_kick_handler); 3201218bdd20SEli Cohen mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_wq"); 32025262912eSEli Cohen if (!mvdev->wq) { 32035262912eSEli Cohen err = -ENOMEM; 32045262912eSEli Cohen goto err_res2; 32055262912eSEli Cohen } 32065262912eSEli Cohen 3207edf747afSEli Cohen ndev->nb.notifier_call = event_handler; 3208edf747afSEli Cohen mlx5_notifier_register(mdev, &ndev->nb); 3209cae15c2eSEli Cohen ndev->nb_registered = true; 321058926c8aSEli Cohen mvdev->vdev.mdev = &mgtdev->mgtdev; 3211acde3929SEli Cohen err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1); 32121a86b377SEli Cohen if (err) 32131a86b377SEli Cohen goto err_reg; 32141a86b377SEli Cohen 321558926c8aSEli Cohen mgtdev->ndev = ndev; 321674c9729dSLeon Romanovsky return 0; 32171a86b377SEli Cohen 32181a86b377SEli Cohen err_reg: 32195262912eSEli Cohen destroy_workqueue(mvdev->wq); 32205262912eSEli Cohen err_res2: 32211a86b377SEli Cohen free_resources(ndev); 32226f5312f8SEli Cohen err_mr: 32236f5312f8SEli Cohen mlx5_vdpa_destroy_mr(mvdev); 32241a86b377SEli Cohen err_res: 32251a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 32267c9f131fSEli Cohen err_mpfs: 32277c9f131fSEli Cohen if (!is_zero_ether_addr(config->mac)) 32287c9f131fSEli Cohen mlx5_mpfs_del_mac(pfmdev, config->mac); 322975560522SEli Cohen err_alloc: 32301a86b377SEli Cohen put_device(&mvdev->vdev.dev); 323174c9729dSLeon Romanovsky return err; 32321a86b377SEli Cohen } 32331a86b377SEli Cohen 323458926c8aSEli Cohen static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev) 323558926c8aSEli Cohen { 323658926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 32375262912eSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(dev); 3238edf747afSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 3239ad6dc1daSEli Cohen struct workqueue_struct *wq; 324058926c8aSEli Cohen 3241cae15c2eSEli Cohen if (ndev->nb_registered) { 3242edf747afSEli Cohen mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 3243cae15c2eSEli Cohen ndev->nb_registered = false; 3244cae15c2eSEli Cohen } 3245ad6dc1daSEli Cohen wq = mvdev->wq; 3246ad6dc1daSEli Cohen mvdev->wq = NULL; 3247ad6dc1daSEli Cohen destroy_workqueue(wq); 324858926c8aSEli Cohen _vdpa_unregister_device(dev); 324958926c8aSEli Cohen mgtdev->ndev = NULL; 325058926c8aSEli Cohen } 325158926c8aSEli Cohen 325258926c8aSEli Cohen static const struct vdpa_mgmtdev_ops mdev_ops = { 325358926c8aSEli Cohen .dev_add = mlx5_vdpa_dev_add, 325458926c8aSEli Cohen .dev_del = mlx5_vdpa_dev_del, 325558926c8aSEli Cohen }; 325658926c8aSEli Cohen 325758926c8aSEli Cohen static struct virtio_device_id id_table[] = { 325858926c8aSEli Cohen { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 325958926c8aSEli Cohen { 0 }, 326058926c8aSEli Cohen }; 326158926c8aSEli Cohen 326258926c8aSEli Cohen static int mlx5v_probe(struct auxiliary_device *adev, 326358926c8aSEli Cohen const struct auxiliary_device_id *id) 326458926c8aSEli Cohen 326558926c8aSEli Cohen { 326658926c8aSEli Cohen struct mlx5_adev *madev = container_of(adev, struct mlx5_adev, adev); 326758926c8aSEli Cohen struct mlx5_core_dev *mdev = madev->mdev; 326858926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev; 326958926c8aSEli Cohen int err; 327058926c8aSEli Cohen 327158926c8aSEli Cohen mgtdev = kzalloc(sizeof(*mgtdev), GFP_KERNEL); 327258926c8aSEli Cohen if (!mgtdev) 327358926c8aSEli Cohen return -ENOMEM; 327458926c8aSEli Cohen 327558926c8aSEli Cohen mgtdev->mgtdev.ops = &mdev_ops; 327658926c8aSEli Cohen mgtdev->mgtdev.device = mdev->device; 327758926c8aSEli Cohen mgtdev->mgtdev.id_table = id_table; 327875560522SEli Cohen mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | 32791e00e821SEli Cohen BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP) | 32801e00e821SEli Cohen BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU); 328179de65edSEli Cohen mgtdev->mgtdev.max_supported_vqs = 328279de65edSEli Cohen MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1; 328379de65edSEli Cohen mgtdev->mgtdev.supported_features = get_supported_features(mdev); 328458926c8aSEli Cohen mgtdev->madev = madev; 328558926c8aSEli Cohen 328658926c8aSEli Cohen err = vdpa_mgmtdev_register(&mgtdev->mgtdev); 328758926c8aSEli Cohen if (err) 328858926c8aSEli Cohen goto reg_err; 328958926c8aSEli Cohen 329045e3a279SDavid E. Box auxiliary_set_drvdata(adev, mgtdev); 329158926c8aSEli Cohen 329258926c8aSEli Cohen return 0; 329358926c8aSEli Cohen 329458926c8aSEli Cohen reg_err: 329558926c8aSEli Cohen kfree(mgtdev); 329658926c8aSEli Cohen return err; 329758926c8aSEli Cohen } 329858926c8aSEli Cohen 329974c9729dSLeon Romanovsky static void mlx5v_remove(struct auxiliary_device *adev) 33001a86b377SEli Cohen { 330158926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev; 330274c9729dSLeon Romanovsky 330345e3a279SDavid E. Box mgtdev = auxiliary_get_drvdata(adev); 330458926c8aSEli Cohen vdpa_mgmtdev_unregister(&mgtdev->mgtdev); 330558926c8aSEli Cohen kfree(mgtdev); 33061a86b377SEli Cohen } 330774c9729dSLeon Romanovsky 330874c9729dSLeon Romanovsky static const struct auxiliary_device_id mlx5v_id_table[] = { 330974c9729dSLeon Romanovsky { .name = MLX5_ADEV_NAME ".vnet", }, 331074c9729dSLeon Romanovsky {}, 331174c9729dSLeon Romanovsky }; 331274c9729dSLeon Romanovsky 331374c9729dSLeon Romanovsky MODULE_DEVICE_TABLE(auxiliary, mlx5v_id_table); 331474c9729dSLeon Romanovsky 331574c9729dSLeon Romanovsky static struct auxiliary_driver mlx5v_driver = { 331674c9729dSLeon Romanovsky .name = "vnet", 331774c9729dSLeon Romanovsky .probe = mlx5v_probe, 331874c9729dSLeon Romanovsky .remove = mlx5v_remove, 331974c9729dSLeon Romanovsky .id_table = mlx5v_id_table, 332074c9729dSLeon Romanovsky }; 332174c9729dSLeon Romanovsky 332274c9729dSLeon Romanovsky module_auxiliary_driver(mlx5v_driver); 3323