11a86b377SEli Cohen // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 21a86b377SEli Cohen /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 31a86b377SEli Cohen 474c9729dSLeon Romanovsky #include <linux/module.h> 51a86b377SEli Cohen #include <linux/vdpa.h> 674c9729dSLeon Romanovsky #include <linux/vringh.h> 774c9729dSLeon Romanovsky #include <uapi/linux/virtio_net.h> 81a86b377SEli Cohen #include <uapi/linux/virtio_ids.h> 9a007d940SEli Cohen #include <uapi/linux/vdpa.h> 101a86b377SEli Cohen #include <linux/virtio_config.h> 1174c9729dSLeon Romanovsky #include <linux/auxiliary_bus.h> 1274c9729dSLeon Romanovsky #include <linux/mlx5/cq.h> 131a86b377SEli Cohen #include <linux/mlx5/qp.h> 141a86b377SEli Cohen #include <linux/mlx5/device.h> 1574c9729dSLeon Romanovsky #include <linux/mlx5/driver.h> 161a86b377SEli Cohen #include <linux/mlx5/vport.h> 171a86b377SEli Cohen #include <linux/mlx5/fs.h> 180aae392bSLeon Romanovsky #include <linux/mlx5/mlx5_ifc_vdpa.h> 197c9f131fSEli Cohen #include <linux/mlx5/mpfs.h> 201a86b377SEli Cohen #include "mlx5_vdpa.h" 211a86b377SEli Cohen 2274c9729dSLeon Romanovsky MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 2374c9729dSLeon Romanovsky MODULE_DESCRIPTION("Mellanox VDPA driver"); 2474c9729dSLeon Romanovsky MODULE_LICENSE("Dual BSD/GPL"); 2574c9729dSLeon Romanovsky 2674c9729dSLeon Romanovsky #define to_mlx5_vdpa_ndev(__mvdev) \ 2774c9729dSLeon Romanovsky container_of(__mvdev, struct mlx5_vdpa_net, mvdev) 281a86b377SEli Cohen #define to_mvdev(__vdev) container_of((__vdev), struct mlx5_vdpa_dev, vdev) 291a86b377SEli Cohen 301a86b377SEli Cohen #define VALID_FEATURES_MASK \ 31cbb52359SNathan Chancellor (BIT_ULL(VIRTIO_NET_F_CSUM) | BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | \ 32cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_MAC) | \ 33cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | \ 34cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | \ 35cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | BIT_ULL(VIRTIO_NET_F_HOST_ECN) | BIT_ULL(VIRTIO_NET_F_HOST_UFO) | \ 36cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | BIT_ULL(VIRTIO_NET_F_STATUS) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | \ 37cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_RX) | BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | \ 38cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE) | \ 39cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | \ 40cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_RSS) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY) | \ 41cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX) | BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY) | \ 42cbb52359SNathan Chancellor BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | \ 43cbb52359SNathan Chancellor BIT_ULL(VIRTIO_F_RING_PACKED) | BIT_ULL(VIRTIO_F_ORDER_PLATFORM) | BIT_ULL(VIRTIO_F_SR_IOV)) 441a86b377SEli Cohen 451a86b377SEli Cohen #define VALID_STATUS_MASK \ 461a86b377SEli Cohen (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \ 471a86b377SEli Cohen VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED) 481a86b377SEli Cohen 49e4fc6650SEli Cohen #define MLX5_FEATURE(_mvdev, _feature) (!!((_mvdev)->actual_features & BIT_ULL(_feature))) 50e4fc6650SEli Cohen 511a86b377SEli Cohen struct mlx5_vdpa_net_resources { 521a86b377SEli Cohen u32 tisn; 531a86b377SEli Cohen u32 tdn; 541a86b377SEli Cohen u32 tirn; 551a86b377SEli Cohen u32 rqtn; 561a86b377SEli Cohen bool valid; 571a86b377SEli Cohen }; 581a86b377SEli Cohen 591a86b377SEli Cohen struct mlx5_vdpa_cq_buf { 601a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 611a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 621a86b377SEli Cohen int cqe_size; 631a86b377SEli Cohen int nent; 641a86b377SEli Cohen }; 651a86b377SEli Cohen 661a86b377SEli Cohen struct mlx5_vdpa_cq { 671a86b377SEli Cohen struct mlx5_core_cq mcq; 681a86b377SEli Cohen struct mlx5_vdpa_cq_buf buf; 691a86b377SEli Cohen struct mlx5_db db; 701a86b377SEli Cohen int cqe; 711a86b377SEli Cohen }; 721a86b377SEli Cohen 731a86b377SEli Cohen struct mlx5_vdpa_umem { 741a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 751a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 761a86b377SEli Cohen int size; 771a86b377SEli Cohen u32 id; 781a86b377SEli Cohen }; 791a86b377SEli Cohen 801a86b377SEli Cohen struct mlx5_vdpa_qp { 811a86b377SEli Cohen struct mlx5_core_qp mqp; 821a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 831a86b377SEli Cohen struct mlx5_db db; 841a86b377SEli Cohen u16 head; 851a86b377SEli Cohen bool fw; 861a86b377SEli Cohen }; 871a86b377SEli Cohen 881a86b377SEli Cohen struct mlx5_vq_restore_info { 891a86b377SEli Cohen u32 num_ent; 901a86b377SEli Cohen u64 desc_addr; 911a86b377SEli Cohen u64 device_addr; 921a86b377SEli Cohen u64 driver_addr; 931a86b377SEli Cohen u16 avail_index; 94b35ccebeSEli Cohen u16 used_index; 951a86b377SEli Cohen bool ready; 961a86b377SEli Cohen bool restore; 971a86b377SEli Cohen }; 981a86b377SEli Cohen 991a86b377SEli Cohen struct mlx5_vdpa_virtqueue { 1001a86b377SEli Cohen bool ready; 1011a86b377SEli Cohen u64 desc_addr; 1021a86b377SEli Cohen u64 device_addr; 1031a86b377SEli Cohen u64 driver_addr; 1041a86b377SEli Cohen u32 num_ent; 1051a86b377SEli Cohen 1061a86b377SEli Cohen /* Resources for implementing the notification channel from the device 1071a86b377SEli Cohen * to the driver. fwqp is the firmware end of an RC connection; the 1081a86b377SEli Cohen * other end is vqqp used by the driver. cq is is where completions are 1091a86b377SEli Cohen * reported. 1101a86b377SEli Cohen */ 1111a86b377SEli Cohen struct mlx5_vdpa_cq cq; 1121a86b377SEli Cohen struct mlx5_vdpa_qp fwqp; 1131a86b377SEli Cohen struct mlx5_vdpa_qp vqqp; 1141a86b377SEli Cohen 1151a86b377SEli Cohen /* umem resources are required for the virtqueue operation. They're use 1161a86b377SEli Cohen * is internal and they must be provided by the driver. 1171a86b377SEli Cohen */ 1181a86b377SEli Cohen struct mlx5_vdpa_umem umem1; 1191a86b377SEli Cohen struct mlx5_vdpa_umem umem2; 1201a86b377SEli Cohen struct mlx5_vdpa_umem umem3; 1211a86b377SEli Cohen 1221892a3d4SEli Cohen u32 counter_set_id; 1231a86b377SEli Cohen bool initialized; 1241a86b377SEli Cohen int index; 1251a86b377SEli Cohen u32 virtq_id; 1261a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 1271a86b377SEli Cohen u16 avail_idx; 128b35ccebeSEli Cohen u16 used_idx; 1291a86b377SEli Cohen int fw_state; 1301a86b377SEli Cohen 1311a86b377SEli Cohen /* keep last in the struct */ 1321a86b377SEli Cohen struct mlx5_vq_restore_info ri; 1331a86b377SEli Cohen }; 1341a86b377SEli Cohen 135e4fc6650SEli Cohen static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx) 136e4fc6650SEli Cohen { 137f8ae3a48SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) { 138f8ae3a48SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 139f8ae3a48SEli Cohen return idx < 2; 140f8ae3a48SEli Cohen else 141f8ae3a48SEli Cohen return idx < 3; 142f8ae3a48SEli Cohen } 143e4fc6650SEli Cohen 144f8ae3a48SEli Cohen return idx <= mvdev->max_idx; 145e4fc6650SEli Cohen } 146e4fc6650SEli Cohen 1471a86b377SEli Cohen struct mlx5_vdpa_net { 1481a86b377SEli Cohen struct mlx5_vdpa_dev mvdev; 1491a86b377SEli Cohen struct mlx5_vdpa_net_resources res; 1501a86b377SEli Cohen struct virtio_net_config config; 15175560522SEli Cohen struct mlx5_vdpa_virtqueue *vqs; 15275560522SEli Cohen struct vdpa_callback *event_cbs; 1531a86b377SEli Cohen 1541a86b377SEli Cohen /* Serialize vq resources creation and destruction. This is required 1551a86b377SEli Cohen * since memory map might change and we need to destroy and create 1561a86b377SEli Cohen * resources while driver in operational. 1571a86b377SEli Cohen */ 158759ae7f9SEli Cohen struct rw_semaphore reslock; 1591a86b377SEli Cohen struct mlx5_flow_table *rxft; 1601a86b377SEli Cohen struct mlx5_fc *rx_counter; 161540061acSEli Cohen struct mlx5_flow_handle *rx_rule_ucast; 162540061acSEli Cohen struct mlx5_flow_handle *rx_rule_mcast; 1631a86b377SEli Cohen bool setup; 1645262912eSEli Cohen u32 cur_num_vqs; 165acde3929SEli Cohen u32 rqt_size; 166edf747afSEli Cohen struct notifier_block nb; 167edf747afSEli Cohen struct vdpa_callback config_cb; 16855ebf0d6SJason Wang struct mlx5_vdpa_wq_ent cvq_ent; 1691a86b377SEli Cohen }; 1701a86b377SEli Cohen 1711a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev); 1721a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev); 173ae0428deSEli Cohen static int setup_driver(struct mlx5_vdpa_dev *mvdev); 1741a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev); 1751a86b377SEli Cohen 1761a86b377SEli Cohen static bool mlx5_vdpa_debug; 1771a86b377SEli Cohen 1785262912eSEli Cohen #define MLX5_CVQ_MAX_ENT 16 1795262912eSEli Cohen 1801a86b377SEli Cohen #define MLX5_LOG_VIO_FLAG(_feature) \ 1811a86b377SEli Cohen do { \ 182cbb52359SNathan Chancellor if (features & BIT_ULL(_feature)) \ 1831a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_feature); \ 1841a86b377SEli Cohen } while (0) 1851a86b377SEli Cohen 1861a86b377SEli Cohen #define MLX5_LOG_VIO_STAT(_status) \ 1871a86b377SEli Cohen do { \ 1881a86b377SEli Cohen if (status & (_status)) \ 1891a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_status); \ 1901a86b377SEli Cohen } while (0) 1911a86b377SEli Cohen 19252893733SEli Cohen /* TODO: cross-endian support */ 19352893733SEli Cohen static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev) 19452893733SEli Cohen { 19552893733SEli Cohen return virtio_legacy_is_little_endian() || 19652893733SEli Cohen (mvdev->actual_features & BIT_ULL(VIRTIO_F_VERSION_1)); 19752893733SEli Cohen } 19852893733SEli Cohen 19952893733SEli Cohen static u16 mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev *mvdev, __virtio16 val) 20052893733SEli Cohen { 20152893733SEli Cohen return __virtio16_to_cpu(mlx5_vdpa_is_little_endian(mvdev), val); 20252893733SEli Cohen } 20352893733SEli Cohen 20452893733SEli Cohen static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val) 20552893733SEli Cohen { 20652893733SEli Cohen return __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), val); 20752893733SEli Cohen } 20852893733SEli Cohen 2095262912eSEli Cohen static u16 ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev) 2105262912eSEli Cohen { 21152893733SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) 21252893733SEli Cohen return 2; 21352893733SEli Cohen 214acde3929SEli Cohen return mvdev->max_vqs; 2155262912eSEli Cohen } 2165262912eSEli Cohen 2175262912eSEli Cohen static bool is_ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev, u16 idx) 2185262912eSEli Cohen { 2195262912eSEli Cohen return idx == ctrl_vq_idx(mvdev); 2205262912eSEli Cohen } 2215262912eSEli Cohen 2221a86b377SEli Cohen static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set) 2231a86b377SEli Cohen { 2241a86b377SEli Cohen if (status & ~VALID_STATUS_MASK) 2251a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n", 2261a86b377SEli Cohen status & ~VALID_STATUS_MASK); 2271a86b377SEli Cohen 2281a86b377SEli Cohen if (!mlx5_vdpa_debug) 2291a86b377SEli Cohen return; 2301a86b377SEli Cohen 2311a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get"); 2321a86b377SEli Cohen if (set && !status) { 2331a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver resets the device\n"); 2341a86b377SEli Cohen return; 2351a86b377SEli Cohen } 2361a86b377SEli Cohen 2371a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE); 2381a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER); 2391a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK); 2401a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK); 2411a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET); 2421a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED); 2431a86b377SEli Cohen } 2441a86b377SEli Cohen 2451a86b377SEli Cohen static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set) 2461a86b377SEli Cohen { 2471a86b377SEli Cohen if (features & ~VALID_FEATURES_MASK) 2481a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n", 2491a86b377SEli Cohen features & ~VALID_FEATURES_MASK); 2501a86b377SEli Cohen 2511a86b377SEli Cohen if (!mlx5_vdpa_debug) 2521a86b377SEli Cohen return; 2531a86b377SEli Cohen 2541a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads"); 2551a86b377SEli Cohen if (!features) 2561a86b377SEli Cohen mlx5_vdpa_info(mvdev, "all feature bits are cleared\n"); 2571a86b377SEli Cohen 2581a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM); 2591a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM); 2601a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 2611a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU); 2621a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC); 2631a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4); 2641a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6); 2651a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN); 2661a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO); 2671a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4); 2681a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6); 2691a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN); 2701a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO); 2711a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF); 2721a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS); 2731a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ); 2741a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX); 2751a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN); 2761a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA); 2771a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE); 2781a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ); 2791a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR); 2801a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT); 2811a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS); 2821a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT); 2831a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY); 2841a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX); 2851a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY); 2861a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT); 2871a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1); 2881a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM); 2891a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED); 2901a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM); 2911a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV); 2921a86b377SEli Cohen } 2931a86b377SEli Cohen 2941a86b377SEli Cohen static int create_tis(struct mlx5_vdpa_net *ndev) 2951a86b377SEli Cohen { 2961a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; 2971a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 2981a86b377SEli Cohen void *tisc; 2991a86b377SEli Cohen int err; 3001a86b377SEli Cohen 3011a86b377SEli Cohen tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3021a86b377SEli Cohen MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); 3031a86b377SEli Cohen err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); 3041a86b377SEli Cohen if (err) 3051a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err); 3061a86b377SEli Cohen 3071a86b377SEli Cohen return err; 3081a86b377SEli Cohen } 3091a86b377SEli Cohen 3101a86b377SEli Cohen static void destroy_tis(struct mlx5_vdpa_net *ndev) 3111a86b377SEli Cohen { 3121a86b377SEli Cohen mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); 3131a86b377SEli Cohen } 3141a86b377SEli Cohen 3151a86b377SEli Cohen #define MLX5_VDPA_CQE_SIZE 64 3161a86b377SEli Cohen #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE) 3171a86b377SEli Cohen 3181a86b377SEli Cohen static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) 3191a86b377SEli Cohen { 3201a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &buf->frag_buf; 3211a86b377SEli Cohen u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE; 3221a86b377SEli Cohen u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE; 3231a86b377SEli Cohen int err; 3241a86b377SEli Cohen 3251a86b377SEli Cohen err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, 3261a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3271a86b377SEli Cohen if (err) 3281a86b377SEli Cohen return err; 3291a86b377SEli Cohen 3301a86b377SEli Cohen mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc); 3311a86b377SEli Cohen 3321a86b377SEli Cohen buf->cqe_size = MLX5_VDPA_CQE_SIZE; 3331a86b377SEli Cohen buf->nent = nent; 3341a86b377SEli Cohen 3351a86b377SEli Cohen return 0; 3361a86b377SEli Cohen } 3371a86b377SEli Cohen 3381a86b377SEli Cohen static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) 3391a86b377SEli Cohen { 3401a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &umem->frag_buf; 3411a86b377SEli Cohen 3421a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, 3431a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3441a86b377SEli Cohen } 3451a86b377SEli Cohen 3461a86b377SEli Cohen static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) 3471a86b377SEli Cohen { 3481a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); 3491a86b377SEli Cohen } 3501a86b377SEli Cohen 3511a86b377SEli Cohen static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n) 3521a86b377SEli Cohen { 3531a86b377SEli Cohen return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n); 3541a86b377SEli Cohen } 3551a86b377SEli Cohen 3561a86b377SEli Cohen static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf) 3571a86b377SEli Cohen { 3581a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 3591a86b377SEli Cohen void *cqe; 3601a86b377SEli Cohen int i; 3611a86b377SEli Cohen 3621a86b377SEli Cohen for (i = 0; i < buf->nent; i++) { 3631a86b377SEli Cohen cqe = get_cqe(vcq, i); 3641a86b377SEli Cohen cqe64 = cqe; 3651a86b377SEli Cohen cqe64->op_own = MLX5_CQE_INVALID << 4; 3661a86b377SEli Cohen } 3671a86b377SEli Cohen } 3681a86b377SEli Cohen 3691a86b377SEli Cohen static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n) 3701a86b377SEli Cohen { 3711a86b377SEli Cohen struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1)); 3721a86b377SEli Cohen 3731a86b377SEli Cohen if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && 3741a86b377SEli Cohen !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe))) 3751a86b377SEli Cohen return cqe64; 3761a86b377SEli Cohen 3771a86b377SEli Cohen return NULL; 3781a86b377SEli Cohen } 3791a86b377SEli Cohen 3801a86b377SEli Cohen static void rx_post(struct mlx5_vdpa_qp *vqp, int n) 3811a86b377SEli Cohen { 3821a86b377SEli Cohen vqp->head += n; 3831a86b377SEli Cohen vqp->db.db[0] = cpu_to_be32(vqp->head); 3841a86b377SEli Cohen } 3851a86b377SEli Cohen 3861a86b377SEli Cohen static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, 3871a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq, u32 num_ent) 3881a86b377SEli Cohen { 3891a86b377SEli Cohen struct mlx5_vdpa_qp *vqp; 3901a86b377SEli Cohen __be64 *pas; 3911a86b377SEli Cohen void *qpc; 3921a86b377SEli Cohen 3931a86b377SEli Cohen vqp = fw ? &mvq->fwqp : &mvq->vqqp; 3941a86b377SEli Cohen MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); 3951a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 3961a86b377SEli Cohen if (vqp->fw) { 3971a86b377SEli Cohen /* Firmware QP is allocated by the driver for the firmware's 3981a86b377SEli Cohen * use so we can skip part of the params as they will be chosen by firmware 3991a86b377SEli Cohen */ 4001a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4011a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ); 4021a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 4031a86b377SEli Cohen return; 4041a86b377SEli Cohen } 4051a86b377SEli Cohen 4061a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4071a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4081a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4091a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4101a86b377SEli Cohen MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); 4111a86b377SEli Cohen MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 4121a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 4131a86b377SEli Cohen MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn); 4141a86b377SEli Cohen MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent)); 4151a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 4161a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas); 4171a86b377SEli Cohen mlx5_fill_page_frag_array(&vqp->frag_buf, pas); 4181a86b377SEli Cohen } 4191a86b377SEli Cohen 4201a86b377SEli Cohen static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) 4211a86b377SEli Cohen { 4221a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, 4231a86b377SEli Cohen num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf, 4241a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 4251a86b377SEli Cohen } 4261a86b377SEli Cohen 4271a86b377SEli Cohen static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 4281a86b377SEli Cohen { 4291a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); 4301a86b377SEli Cohen } 4311a86b377SEli Cohen 4321a86b377SEli Cohen static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 4331a86b377SEli Cohen struct mlx5_vdpa_qp *vqp) 4341a86b377SEli Cohen { 4351a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 4361a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_qp_in); 4371a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 4381a86b377SEli Cohen void *qpc; 4391a86b377SEli Cohen void *in; 4401a86b377SEli Cohen int err; 4411a86b377SEli Cohen 4421a86b377SEli Cohen if (!vqp->fw) { 4431a86b377SEli Cohen vqp = &mvq->vqqp; 4441a86b377SEli Cohen err = rq_buf_alloc(ndev, vqp, mvq->num_ent); 4451a86b377SEli Cohen if (err) 4461a86b377SEli Cohen return err; 4471a86b377SEli Cohen 4481a86b377SEli Cohen err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); 4491a86b377SEli Cohen if (err) 4501a86b377SEli Cohen goto err_db; 4511a86b377SEli Cohen inlen += vqp->frag_buf.npages * sizeof(__be64); 4521a86b377SEli Cohen } 4531a86b377SEli Cohen 4541a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 4551a86b377SEli Cohen if (!in) { 4561a86b377SEli Cohen err = -ENOMEM; 4571a86b377SEli Cohen goto err_kzalloc; 4581a86b377SEli Cohen } 4591a86b377SEli Cohen 4601a86b377SEli Cohen qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); 4611a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4621a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4631a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4641a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4651a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4661a86b377SEli Cohen if (!vqp->fw) 4671a86b377SEli Cohen MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma); 4681a86b377SEli Cohen MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 4691a86b377SEli Cohen err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 4701a86b377SEli Cohen kfree(in); 4711a86b377SEli Cohen if (err) 4721a86b377SEli Cohen goto err_kzalloc; 4731a86b377SEli Cohen 4741a86b377SEli Cohen vqp->mqp.uid = ndev->mvdev.res.uid; 4751a86b377SEli Cohen vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn); 4761a86b377SEli Cohen 4771a86b377SEli Cohen if (!vqp->fw) 4781a86b377SEli Cohen rx_post(vqp, mvq->num_ent); 4791a86b377SEli Cohen 4801a86b377SEli Cohen return 0; 4811a86b377SEli Cohen 4821a86b377SEli Cohen err_kzalloc: 4831a86b377SEli Cohen if (!vqp->fw) 4841a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 4851a86b377SEli Cohen err_db: 4861a86b377SEli Cohen if (!vqp->fw) 4871a86b377SEli Cohen rq_buf_free(ndev, vqp); 4881a86b377SEli Cohen 4891a86b377SEli Cohen return err; 4901a86b377SEli Cohen } 4911a86b377SEli Cohen 4921a86b377SEli Cohen static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 4931a86b377SEli Cohen { 4941a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 4951a86b377SEli Cohen 4961a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 4971a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn); 4981a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); 4991a86b377SEli Cohen if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) 5001a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); 5011a86b377SEli Cohen if (!vqp->fw) { 5021a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 5031a86b377SEli Cohen rq_buf_free(ndev, vqp); 5041a86b377SEli Cohen } 5051a86b377SEli Cohen } 5061a86b377SEli Cohen 5071a86b377SEli Cohen static void *next_cqe_sw(struct mlx5_vdpa_cq *cq) 5081a86b377SEli Cohen { 5091a86b377SEli Cohen return get_sw_cqe(cq, cq->mcq.cons_index); 5101a86b377SEli Cohen } 5111a86b377SEli Cohen 5121a86b377SEli Cohen static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq) 5131a86b377SEli Cohen { 5141a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 5151a86b377SEli Cohen 5161a86b377SEli Cohen cqe64 = next_cqe_sw(vcq); 5171a86b377SEli Cohen if (!cqe64) 5181a86b377SEli Cohen return -EAGAIN; 5191a86b377SEli Cohen 5201a86b377SEli Cohen vcq->mcq.cons_index++; 5211a86b377SEli Cohen return 0; 5221a86b377SEli Cohen } 5231a86b377SEli Cohen 5241a86b377SEli Cohen static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num) 5251a86b377SEli Cohen { 526db296d25SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 527db296d25SEli Cohen struct vdpa_callback *event_cb; 528db296d25SEli Cohen 529db296d25SEli Cohen event_cb = &ndev->event_cbs[mvq->index]; 5301a86b377SEli Cohen mlx5_cq_set_ci(&mvq->cq.mcq); 53183ef73b2SEli Cohen 53283ef73b2SEli Cohen /* make sure CQ cosumer update is visible to the hardware before updating 53383ef73b2SEli Cohen * RX doorbell record. 53483ef73b2SEli Cohen */ 53583ef73b2SEli Cohen dma_wmb(); 5361a86b377SEli Cohen rx_post(&mvq->vqqp, num); 537db296d25SEli Cohen if (event_cb->callback) 538db296d25SEli Cohen event_cb->callback(event_cb->private); 5391a86b377SEli Cohen } 5401a86b377SEli Cohen 5411a86b377SEli Cohen static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) 5421a86b377SEli Cohen { 5431a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq); 5441a86b377SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 5451a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5461a86b377SEli Cohen int num = 0; 5471a86b377SEli Cohen 5481a86b377SEli Cohen while (!mlx5_vdpa_poll_one(&mvq->cq)) { 5491a86b377SEli Cohen num++; 5501a86b377SEli Cohen if (num > mvq->num_ent / 2) { 5511a86b377SEli Cohen /* If completions keep coming while we poll, we want to 5521a86b377SEli Cohen * let the hardware know that we consumed them by 5531a86b377SEli Cohen * updating the doorbell record. We also let vdpa core 5541a86b377SEli Cohen * know about this so it passes it on the virtio driver 5551a86b377SEli Cohen * on the guest. 5561a86b377SEli Cohen */ 5571a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 5581a86b377SEli Cohen num = 0; 5591a86b377SEli Cohen } 5601a86b377SEli Cohen } 5611a86b377SEli Cohen 5621a86b377SEli Cohen if (num) 5631a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 5641a86b377SEli Cohen 5651a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 5661a86b377SEli Cohen } 5671a86b377SEli Cohen 5681a86b377SEli Cohen static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) 5691a86b377SEli Cohen { 5701a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 5711a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 5721a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5731a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 5741a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 5751a86b377SEli Cohen __be64 *pas; 5761a86b377SEli Cohen int inlen; 5771a86b377SEli Cohen void *cqc; 5781a86b377SEli Cohen void *in; 5791a86b377SEli Cohen int err; 5801a86b377SEli Cohen int eqn; 5811a86b377SEli Cohen 5821a86b377SEli Cohen err = mlx5_db_alloc(mdev, &vcq->db); 5831a86b377SEli Cohen if (err) 5841a86b377SEli Cohen return err; 5851a86b377SEli Cohen 5861a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 5871a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 5881a86b377SEli Cohen vcq->mcq.cqe_sz = 64; 5891a86b377SEli Cohen 5901a86b377SEli Cohen err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); 5911a86b377SEli Cohen if (err) 5921a86b377SEli Cohen goto err_db; 5931a86b377SEli Cohen 5941a86b377SEli Cohen cq_frag_buf_init(vcq, &vcq->buf); 5951a86b377SEli Cohen 5961a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 5971a86b377SEli Cohen MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages; 5981a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 5991a86b377SEli Cohen if (!in) { 6001a86b377SEli Cohen err = -ENOMEM; 6011a86b377SEli Cohen goto err_vzalloc; 6021a86b377SEli Cohen } 6031a86b377SEli Cohen 6041a86b377SEli Cohen MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); 6051a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 6061a86b377SEli Cohen mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas); 6071a86b377SEli Cohen 6081a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 6091a86b377SEli Cohen MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 6101a86b377SEli Cohen 6111a86b377SEli Cohen /* Use vector 0 by default. Consider adding code to choose least used 6121a86b377SEli Cohen * vector. 6131a86b377SEli Cohen */ 614563476aeSShay Drory err = mlx5_vector2eqn(mdev, 0, &eqn); 6151a86b377SEli Cohen if (err) 6161a86b377SEli Cohen goto err_vec; 6171a86b377SEli Cohen 6181a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 6191a86b377SEli Cohen MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent)); 6201a86b377SEli Cohen MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); 621616d5769STal Gilboa MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 6221a86b377SEli Cohen MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma); 6231a86b377SEli Cohen 6241a86b377SEli Cohen err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out)); 6251a86b377SEli Cohen if (err) 6261a86b377SEli Cohen goto err_vec; 6271a86b377SEli Cohen 6281a86b377SEli Cohen vcq->mcq.comp = mlx5_vdpa_cq_comp; 6291a86b377SEli Cohen vcq->cqe = num_ent; 6301a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 6311a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 6321a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 6331a86b377SEli Cohen kfree(in); 6341a86b377SEli Cohen return 0; 6351a86b377SEli Cohen 6361a86b377SEli Cohen err_vec: 6371a86b377SEli Cohen kfree(in); 6381a86b377SEli Cohen err_vzalloc: 6391a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 6401a86b377SEli Cohen err_db: 6411a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 6421a86b377SEli Cohen return err; 6431a86b377SEli Cohen } 6441a86b377SEli Cohen 6451a86b377SEli Cohen static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) 6461a86b377SEli Cohen { 6471a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 6481a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 6491a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 6501a86b377SEli Cohen 6511a86b377SEli Cohen if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) { 6521a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); 6531a86b377SEli Cohen return; 6541a86b377SEli Cohen } 6551a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 6561a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 6571a86b377SEli Cohen } 6581a86b377SEli Cohen 65971ab6a7cSEli Cohen static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, 6601a86b377SEli Cohen struct mlx5_vdpa_umem **umemp) 6611a86b377SEli Cohen { 6621a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 6631a86b377SEli Cohen int p_a; 6641a86b377SEli Cohen int p_b; 6651a86b377SEli Cohen 6661a86b377SEli Cohen switch (num) { 6671a86b377SEli Cohen case 1: 6681a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a); 6691a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b); 6701a86b377SEli Cohen *umemp = &mvq->umem1; 6711a86b377SEli Cohen break; 6721a86b377SEli Cohen case 2: 6731a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a); 6741a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b); 6751a86b377SEli Cohen *umemp = &mvq->umem2; 6761a86b377SEli Cohen break; 6771a86b377SEli Cohen case 3: 6781a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a); 6791a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b); 6801a86b377SEli Cohen *umemp = &mvq->umem3; 6811a86b377SEli Cohen break; 6821a86b377SEli Cohen } 68371ab6a7cSEli Cohen (*umemp)->size = p_a * mvq->num_ent + p_b; 6841a86b377SEli Cohen } 6851a86b377SEli Cohen 6861a86b377SEli Cohen static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) 6871a86b377SEli Cohen { 6881a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); 6891a86b377SEli Cohen } 6901a86b377SEli Cohen 6911a86b377SEli Cohen static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 6921a86b377SEli Cohen { 6931a86b377SEli Cohen int inlen; 6941a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {}; 6951a86b377SEli Cohen void *um; 6961a86b377SEli Cohen void *in; 6971a86b377SEli Cohen int err; 6981a86b377SEli Cohen __be64 *pas; 6991a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 7001a86b377SEli Cohen 70171ab6a7cSEli Cohen set_umem_size(ndev, mvq, num, &umem); 70271ab6a7cSEli Cohen err = umem_frag_buf_alloc(ndev, umem, umem->size); 7031a86b377SEli Cohen if (err) 7041a86b377SEli Cohen return err; 7051a86b377SEli Cohen 7061a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages; 7071a86b377SEli Cohen 7081a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 7091a86b377SEli Cohen if (!in) { 7101a86b377SEli Cohen err = -ENOMEM; 7111a86b377SEli Cohen goto err_in; 7121a86b377SEli Cohen } 7131a86b377SEli Cohen 7141a86b377SEli Cohen MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM); 7151a86b377SEli Cohen MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); 7161a86b377SEli Cohen um = MLX5_ADDR_OF(create_umem_in, in, umem); 7171a86b377SEli Cohen MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 7181a86b377SEli Cohen MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages); 7191a86b377SEli Cohen 7201a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]); 7211a86b377SEli Cohen mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW); 7221a86b377SEli Cohen 7231a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 7241a86b377SEli Cohen if (err) { 7251a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); 7261a86b377SEli Cohen goto err_cmd; 7271a86b377SEli Cohen } 7281a86b377SEli Cohen 7291a86b377SEli Cohen kfree(in); 7301a86b377SEli Cohen umem->id = MLX5_GET(create_umem_out, out, umem_id); 7311a86b377SEli Cohen 7321a86b377SEli Cohen return 0; 7331a86b377SEli Cohen 7341a86b377SEli Cohen err_cmd: 7351a86b377SEli Cohen kfree(in); 7361a86b377SEli Cohen err_in: 7371a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7381a86b377SEli Cohen return err; 7391a86b377SEli Cohen } 7401a86b377SEli Cohen 7411a86b377SEli Cohen static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 7421a86b377SEli Cohen { 7431a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {}; 7441a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {}; 7451a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 7461a86b377SEli Cohen 7471a86b377SEli Cohen switch (num) { 7481a86b377SEli Cohen case 1: 7491a86b377SEli Cohen umem = &mvq->umem1; 7501a86b377SEli Cohen break; 7511a86b377SEli Cohen case 2: 7521a86b377SEli Cohen umem = &mvq->umem2; 7531a86b377SEli Cohen break; 7541a86b377SEli Cohen case 3: 7551a86b377SEli Cohen umem = &mvq->umem3; 7561a86b377SEli Cohen break; 7571a86b377SEli Cohen } 7581a86b377SEli Cohen 7591a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM); 7601a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, umem_id, umem->id); 7611a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 7621a86b377SEli Cohen return; 7631a86b377SEli Cohen 7641a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7651a86b377SEli Cohen } 7661a86b377SEli Cohen 7671a86b377SEli Cohen static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7681a86b377SEli Cohen { 7691a86b377SEli Cohen int num; 7701a86b377SEli Cohen int err; 7711a86b377SEli Cohen 7721a86b377SEli Cohen for (num = 1; num <= 3; num++) { 7731a86b377SEli Cohen err = create_umem(ndev, mvq, num); 7741a86b377SEli Cohen if (err) 7751a86b377SEli Cohen goto err_umem; 7761a86b377SEli Cohen } 7771a86b377SEli Cohen return 0; 7781a86b377SEli Cohen 7791a86b377SEli Cohen err_umem: 7801a86b377SEli Cohen for (num--; num > 0; num--) 7811a86b377SEli Cohen umem_destroy(ndev, mvq, num); 7821a86b377SEli Cohen 7831a86b377SEli Cohen return err; 7841a86b377SEli Cohen } 7851a86b377SEli Cohen 7861a86b377SEli Cohen static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7871a86b377SEli Cohen { 7881a86b377SEli Cohen int num; 7891a86b377SEli Cohen 7901a86b377SEli Cohen for (num = 3; num > 0; num--) 7911a86b377SEli Cohen umem_destroy(ndev, mvq, num); 7921a86b377SEli Cohen } 7931a86b377SEli Cohen 7941a86b377SEli Cohen static int get_queue_type(struct mlx5_vdpa_net *ndev) 7951a86b377SEli Cohen { 7961a86b377SEli Cohen u32 type_mask; 7971a86b377SEli Cohen 7981a86b377SEli Cohen type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); 7991a86b377SEli Cohen 8001a86b377SEli Cohen /* prefer split queue */ 801879753c8SEli Cohen if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT) 8021a86b377SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT; 803879753c8SEli Cohen 804879753c8SEli Cohen WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED)); 805879753c8SEli Cohen 806879753c8SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED; 8071a86b377SEli Cohen } 8081a86b377SEli Cohen 8091a86b377SEli Cohen static bool vq_is_tx(u16 idx) 8101a86b377SEli Cohen { 8111a86b377SEli Cohen return idx % 2; 8121a86b377SEli Cohen } 8131a86b377SEli Cohen 8141a86b377SEli Cohen static u16 get_features_12_3(u64 features) 8151a86b377SEli Cohen { 816cbb52359SNathan Chancellor return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) | 817cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) | 818cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) | 819cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6); 8201a86b377SEli Cohen } 8211a86b377SEli Cohen 8221892a3d4SEli Cohen static bool counters_supported(const struct mlx5_vdpa_dev *mvdev) 8231892a3d4SEli Cohen { 8241892a3d4SEli Cohen return MLX5_CAP_GEN_64(mvdev->mdev, general_obj_types) & 8251892a3d4SEli Cohen BIT_ULL(MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 8261892a3d4SEli Cohen } 8271892a3d4SEli Cohen 8281a86b377SEli Cohen static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 8291a86b377SEli Cohen { 8301a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); 8311a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {}; 8321a86b377SEli Cohen void *obj_context; 8331a86b377SEli Cohen void *cmd_hdr; 8341a86b377SEli Cohen void *vq_ctx; 8351a86b377SEli Cohen void *in; 8361a86b377SEli Cohen int err; 8371a86b377SEli Cohen 8381a86b377SEli Cohen err = umems_create(ndev, mvq); 8391a86b377SEli Cohen if (err) 8401a86b377SEli Cohen return err; 8411a86b377SEli Cohen 8421a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 8431a86b377SEli Cohen if (!in) { 8441a86b377SEli Cohen err = -ENOMEM; 8451a86b377SEli Cohen goto err_alloc; 8461a86b377SEli Cohen } 8471a86b377SEli Cohen 8481a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr); 8491a86b377SEli Cohen 8501a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 8511a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 8521a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 8531a86b377SEli Cohen 8541a86b377SEli Cohen obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context); 8551a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); 856b35ccebeSEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); 8571a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, 8581a86b377SEli Cohen get_features_12_3(ndev->mvdev.actual_features)); 8591a86b377SEli Cohen vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context); 8601a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); 8611a86b377SEli Cohen 8621a86b377SEli Cohen if (vq_is_tx(mvq->index)) 8631a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); 8641a86b377SEli Cohen 8651a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 8661a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); 8671a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 8681a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); 8691a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 8704b454a82SEli Cohen !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); 8711a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); 8721a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); 8731a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); 87483fec3f1SAharon Landau MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey); 8751a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); 8761a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); 8771a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id); 878e3011776SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem2.size); 8791a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id); 880e3011776SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem3.size); 8811a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); 8821892a3d4SEli Cohen if (counters_supported(&ndev->mvdev)) 8831892a3d4SEli Cohen MLX5_SET(virtio_q, vq_ctx, counter_set_id, mvq->counter_set_id); 8841a86b377SEli Cohen 8851a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 8861a86b377SEli Cohen if (err) 8871a86b377SEli Cohen goto err_cmd; 8881a86b377SEli Cohen 8891a86b377SEli Cohen kfree(in); 8901a86b377SEli Cohen mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 8911a86b377SEli Cohen 8921a86b377SEli Cohen return 0; 8931a86b377SEli Cohen 8941a86b377SEli Cohen err_cmd: 8951a86b377SEli Cohen kfree(in); 8961a86b377SEli Cohen err_alloc: 8971a86b377SEli Cohen umems_destroy(ndev, mvq); 8981a86b377SEli Cohen return err; 8991a86b377SEli Cohen } 9001a86b377SEli Cohen 9011a86b377SEli Cohen static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 9021a86b377SEli Cohen { 9031a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {}; 9041a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {}; 9051a86b377SEli Cohen 9061a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode, 9071a86b377SEli Cohen MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 9081a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id); 9091a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); 9101a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type, 9111a86b377SEli Cohen MLX5_OBJ_TYPE_VIRTIO_NET_Q); 9121a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { 9131a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); 9141a86b377SEli Cohen return; 9151a86b377SEli Cohen } 9161a86b377SEli Cohen umems_destroy(ndev, mvq); 9171a86b377SEli Cohen } 9181a86b377SEli Cohen 9191a86b377SEli Cohen static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 9201a86b377SEli Cohen { 9211a86b377SEli Cohen return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn; 9221a86b377SEli Cohen } 9231a86b377SEli Cohen 9241a86b377SEli Cohen static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 9251a86b377SEli Cohen { 9261a86b377SEli Cohen return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn; 9271a86b377SEli Cohen } 9281a86b377SEli Cohen 9291a86b377SEli Cohen static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, 9301a86b377SEli Cohen int *outlen, u32 qpn, u32 rqpn) 9311a86b377SEli Cohen { 9321a86b377SEli Cohen void *qpc; 9331a86b377SEli Cohen void *pp; 9341a86b377SEli Cohen 9351a86b377SEli Cohen switch (cmd) { 9361a86b377SEli Cohen case MLX5_CMD_OP_2RST_QP: 9371a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in); 9381a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out); 9391a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9401a86b377SEli Cohen *out = kzalloc(*outlen, GFP_KERNEL); 941f31231bfSColin Ian King if (!*in || !*out) 9421a86b377SEli Cohen goto outerr; 9431a86b377SEli Cohen 9441a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, opcode, cmd); 9451a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); 9461a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, qpn, qpn); 9471a86b377SEli Cohen break; 9481a86b377SEli Cohen case MLX5_CMD_OP_RST2INIT_QP: 9491a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in); 9501a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out); 9511a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9521a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL); 953f31231bfSColin Ian King if (!*in || !*out) 9541a86b377SEli Cohen goto outerr; 9551a86b377SEli Cohen 9561a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, opcode, cmd); 9571a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); 9581a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, qpn, qpn); 9591a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9601a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9611a86b377SEli Cohen MLX5_SET(qpc, qpc, rwe, 1); 9621a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9631a86b377SEli Cohen MLX5_SET(ads, pp, vhca_port_num, 1); 9641a86b377SEli Cohen break; 9651a86b377SEli Cohen case MLX5_CMD_OP_INIT2RTR_QP: 9661a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in); 9671a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out); 9681a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9691a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL); 970f31231bfSColin Ian King if (!*in || !*out) 9711a86b377SEli Cohen goto outerr; 9721a86b377SEli Cohen 9731a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, opcode, cmd); 9741a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); 9751a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, qpn, qpn); 9761a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9771a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 9781a86b377SEli Cohen MLX5_SET(qpc, qpc, log_msg_max, 30); 9791a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9801a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9811a86b377SEli Cohen MLX5_SET(ads, pp, fl, 1); 9821a86b377SEli Cohen break; 9831a86b377SEli Cohen case MLX5_CMD_OP_RTR2RTS_QP: 9841a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in); 9851a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out); 9861a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9871a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL); 988f31231bfSColin Ian King if (!*in || !*out) 9891a86b377SEli Cohen goto outerr; 9901a86b377SEli Cohen 9911a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd); 9921a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); 9931a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn); 9941a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9951a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9961a86b377SEli Cohen MLX5_SET(ads, pp, ack_timeout, 14); 9971a86b377SEli Cohen MLX5_SET(qpc, qpc, retry_count, 7); 9981a86b377SEli Cohen MLX5_SET(qpc, qpc, rnr_retry, 7); 9991a86b377SEli Cohen break; 10001a86b377SEli Cohen default: 1001f31231bfSColin Ian King goto outerr_nullify; 10021a86b377SEli Cohen } 10031a86b377SEli Cohen 10041a86b377SEli Cohen return; 10051a86b377SEli Cohen 10061a86b377SEli Cohen outerr: 10071a86b377SEli Cohen kfree(*in); 10081a86b377SEli Cohen kfree(*out); 1009f31231bfSColin Ian King outerr_nullify: 10101a86b377SEli Cohen *in = NULL; 10111a86b377SEli Cohen *out = NULL; 10121a86b377SEli Cohen } 10131a86b377SEli Cohen 10141a86b377SEli Cohen static void free_inout(void *in, void *out) 10151a86b377SEli Cohen { 10161a86b377SEli Cohen kfree(in); 10171a86b377SEli Cohen kfree(out); 10181a86b377SEli Cohen } 10191a86b377SEli Cohen 10201a86b377SEli Cohen /* Two QPs are used by each virtqueue. One is used by the driver and one by 10211a86b377SEli Cohen * firmware. The fw argument indicates whether the subjected QP is the one used 10221a86b377SEli Cohen * by firmware. 10231a86b377SEli Cohen */ 10241a86b377SEli Cohen static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) 10251a86b377SEli Cohen { 10261a86b377SEli Cohen int outlen; 10271a86b377SEli Cohen int inlen; 10281a86b377SEli Cohen void *out; 10291a86b377SEli Cohen void *in; 10301a86b377SEli Cohen int err; 10311a86b377SEli Cohen 10321a86b377SEli Cohen alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); 10331a86b377SEli Cohen if (!in || !out) 10341a86b377SEli Cohen return -ENOMEM; 10351a86b377SEli Cohen 10361a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); 10371a86b377SEli Cohen free_inout(in, out); 10381a86b377SEli Cohen return err; 10391a86b377SEli Cohen } 10401a86b377SEli Cohen 10411a86b377SEli Cohen static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 10421a86b377SEli Cohen { 10431a86b377SEli Cohen int err; 10441a86b377SEli Cohen 10451a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); 10461a86b377SEli Cohen if (err) 10471a86b377SEli Cohen return err; 10481a86b377SEli Cohen 10491a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); 10501a86b377SEli Cohen if (err) 10511a86b377SEli Cohen return err; 10521a86b377SEli Cohen 10531a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); 10541a86b377SEli Cohen if (err) 10551a86b377SEli Cohen return err; 10561a86b377SEli Cohen 10571a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); 10581a86b377SEli Cohen if (err) 10591a86b377SEli Cohen return err; 10601a86b377SEli Cohen 10611a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); 10621a86b377SEli Cohen if (err) 10631a86b377SEli Cohen return err; 10641a86b377SEli Cohen 10651a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); 10661a86b377SEli Cohen if (err) 10671a86b377SEli Cohen return err; 10681a86b377SEli Cohen 10691a86b377SEli Cohen return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); 10701a86b377SEli Cohen } 10711a86b377SEli Cohen 10721a86b377SEli Cohen struct mlx5_virtq_attr { 10731a86b377SEli Cohen u8 state; 10741a86b377SEli Cohen u16 available_index; 1075b35ccebeSEli Cohen u16 used_index; 10761a86b377SEli Cohen }; 10771a86b377SEli Cohen 10781a86b377SEli Cohen static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 10791a86b377SEli Cohen struct mlx5_virtq_attr *attr) 10801a86b377SEli Cohen { 10811a86b377SEli Cohen int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out); 10821a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {}; 10831a86b377SEli Cohen void *out; 10841a86b377SEli Cohen void *obj_context; 10851a86b377SEli Cohen void *cmd_hdr; 10861a86b377SEli Cohen int err; 10871a86b377SEli Cohen 10881a86b377SEli Cohen out = kzalloc(outlen, GFP_KERNEL); 10891a86b377SEli Cohen if (!out) 10901a86b377SEli Cohen return -ENOMEM; 10911a86b377SEli Cohen 10921a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr); 10931a86b377SEli Cohen 10941a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 10951a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 10961a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 10971a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 10981a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen); 10991a86b377SEli Cohen if (err) 11001a86b377SEli Cohen goto err_cmd; 11011a86b377SEli Cohen 11021a86b377SEli Cohen obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context); 11031a86b377SEli Cohen memset(attr, 0, sizeof(*attr)); 11041a86b377SEli Cohen attr->state = MLX5_GET(virtio_net_q_object, obj_context, state); 11051a86b377SEli Cohen attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index); 1106b35ccebeSEli Cohen attr->used_index = MLX5_GET(virtio_net_q_object, obj_context, hw_used_index); 11071a86b377SEli Cohen kfree(out); 11081a86b377SEli Cohen return 0; 11091a86b377SEli Cohen 11101a86b377SEli Cohen err_cmd: 11111a86b377SEli Cohen kfree(out); 11121a86b377SEli Cohen return err; 11131a86b377SEli Cohen } 11141a86b377SEli Cohen 11151a86b377SEli Cohen static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) 11161a86b377SEli Cohen { 11171a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); 11181a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {}; 11191a86b377SEli Cohen void *obj_context; 11201a86b377SEli Cohen void *cmd_hdr; 11211a86b377SEli Cohen void *in; 11221a86b377SEli Cohen int err; 11231a86b377SEli Cohen 11241a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 11251a86b377SEli Cohen if (!in) 11261a86b377SEli Cohen return -ENOMEM; 11271a86b377SEli Cohen 11281a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr); 11291a86b377SEli Cohen 11301a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); 11311a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 11321a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 11331a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 11341a86b377SEli Cohen 11351a86b377SEli Cohen obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context); 11361a86b377SEli Cohen MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select, 11371a86b377SEli Cohen MLX5_VIRTQ_MODIFY_MASK_STATE); 11381a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, state, state); 11391a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 11401a86b377SEli Cohen kfree(in); 11411a86b377SEli Cohen if (!err) 11421a86b377SEli Cohen mvq->fw_state = state; 11431a86b377SEli Cohen 11441a86b377SEli Cohen return err; 11451a86b377SEli Cohen } 11461a86b377SEli Cohen 11471892a3d4SEli Cohen static int counter_set_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11481892a3d4SEli Cohen { 11491892a3d4SEli Cohen u32 in[MLX5_ST_SZ_DW(create_virtio_q_counters_in)] = {}; 11501892a3d4SEli Cohen u32 out[MLX5_ST_SZ_DW(create_virtio_q_counters_out)] = {}; 11511892a3d4SEli Cohen void *cmd_hdr; 11521892a3d4SEli Cohen int err; 11531892a3d4SEli Cohen 11541892a3d4SEli Cohen if (!counters_supported(&ndev->mvdev)) 11551892a3d4SEli Cohen return 0; 11561892a3d4SEli Cohen 11571892a3d4SEli Cohen cmd_hdr = MLX5_ADDR_OF(create_virtio_q_counters_in, in, hdr); 11581892a3d4SEli Cohen 11591892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 11601892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 11611892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 11621892a3d4SEli Cohen 11631892a3d4SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 11641892a3d4SEli Cohen if (err) 11651892a3d4SEli Cohen return err; 11661892a3d4SEli Cohen 11671892a3d4SEli Cohen mvq->counter_set_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 11681892a3d4SEli Cohen 11691892a3d4SEli Cohen return 0; 11701892a3d4SEli Cohen } 11711892a3d4SEli Cohen 11721892a3d4SEli Cohen static void counter_set_dealloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11731892a3d4SEli Cohen { 11741892a3d4SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_virtio_q_counters_in)] = {}; 11751892a3d4SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_virtio_q_counters_out)] = {}; 11761892a3d4SEli Cohen 11771892a3d4SEli Cohen if (!counters_supported(&ndev->mvdev)) 11781892a3d4SEli Cohen return; 11791892a3d4SEli Cohen 11801892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 11811892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_id, mvq->counter_set_id); 11821892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.uid, ndev->mvdev.res.uid); 11831892a3d4SEli Cohen MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 11841892a3d4SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 11851892a3d4SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "dealloc counter set 0x%x\n", mvq->counter_set_id); 11861892a3d4SEli Cohen } 11871892a3d4SEli Cohen 11881a86b377SEli Cohen static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11891a86b377SEli Cohen { 11901a86b377SEli Cohen u16 idx = mvq->index; 11911a86b377SEli Cohen int err; 11921a86b377SEli Cohen 11931a86b377SEli Cohen if (!mvq->num_ent) 11941a86b377SEli Cohen return 0; 11951a86b377SEli Cohen 119652893733SEli Cohen if (mvq->initialized) 119752893733SEli Cohen return 0; 11981a86b377SEli Cohen 11991a86b377SEli Cohen err = cq_create(ndev, idx, mvq->num_ent); 12001a86b377SEli Cohen if (err) 12011a86b377SEli Cohen return err; 12021a86b377SEli Cohen 12031a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->fwqp); 12041a86b377SEli Cohen if (err) 12051a86b377SEli Cohen goto err_fwqp; 12061a86b377SEli Cohen 12071a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->vqqp); 12081a86b377SEli Cohen if (err) 12091a86b377SEli Cohen goto err_vqqp; 12101a86b377SEli Cohen 12111a86b377SEli Cohen err = connect_qps(ndev, mvq); 12121a86b377SEli Cohen if (err) 12131a86b377SEli Cohen goto err_connect; 12141a86b377SEli Cohen 12151892a3d4SEli Cohen err = counter_set_alloc(ndev, mvq); 12161892a3d4SEli Cohen if (err) 12171892a3d4SEli Cohen goto err_counter; 12181892a3d4SEli Cohen 12191a86b377SEli Cohen err = create_virtqueue(ndev, mvq); 12201a86b377SEli Cohen if (err) 12211a86b377SEli Cohen goto err_connect; 12221a86b377SEli Cohen 12231a86b377SEli Cohen if (mvq->ready) { 12241a86b377SEli Cohen err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 12251a86b377SEli Cohen if (err) { 12261a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", 12271a86b377SEli Cohen idx, err); 12281a86b377SEli Cohen goto err_connect; 12291a86b377SEli Cohen } 12301a86b377SEli Cohen } 12311a86b377SEli Cohen 12321a86b377SEli Cohen mvq->initialized = true; 12331a86b377SEli Cohen return 0; 12341a86b377SEli Cohen 12351a86b377SEli Cohen err_connect: 12361892a3d4SEli Cohen counter_set_dealloc(ndev, mvq); 12371892a3d4SEli Cohen err_counter: 12381a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 12391a86b377SEli Cohen err_vqqp: 12401a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 12411a86b377SEli Cohen err_fwqp: 12421a86b377SEli Cohen cq_destroy(ndev, idx); 12431a86b377SEli Cohen return err; 12441a86b377SEli Cohen } 12451a86b377SEli Cohen 12461a86b377SEli Cohen static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 12471a86b377SEli Cohen { 12481a86b377SEli Cohen struct mlx5_virtq_attr attr; 12491a86b377SEli Cohen 12501a86b377SEli Cohen if (!mvq->initialized) 12511a86b377SEli Cohen return; 12521a86b377SEli Cohen 12531a86b377SEli Cohen if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 12541a86b377SEli Cohen return; 12551a86b377SEli Cohen 12561a86b377SEli Cohen if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) 12571a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); 12583176e974SSi-Wei Liu 12593176e974SSi-Wei Liu if (query_virtqueue(ndev, mvq, &attr)) { 12603176e974SSi-Wei Liu mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); 12613176e974SSi-Wei Liu return; 12623176e974SSi-Wei Liu } 12633176e974SSi-Wei Liu mvq->avail_idx = attr.available_index; 1264bc04d93eSEli Cohen mvq->used_idx = attr.used_index; 12651a86b377SEli Cohen } 12661a86b377SEli Cohen 12671a86b377SEli Cohen static void suspend_vqs(struct mlx5_vdpa_net *ndev) 12681a86b377SEli Cohen { 12691a86b377SEli Cohen int i; 12701a86b377SEli Cohen 127175560522SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 12721a86b377SEli Cohen suspend_vq(ndev, &ndev->vqs[i]); 12731a86b377SEli Cohen } 12741a86b377SEli Cohen 12751a86b377SEli Cohen static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 12761a86b377SEli Cohen { 12771a86b377SEli Cohen if (!mvq->initialized) 12781a86b377SEli Cohen return; 12791a86b377SEli Cohen 12801a86b377SEli Cohen suspend_vq(ndev, mvq); 12811a86b377SEli Cohen destroy_virtqueue(ndev, mvq); 12821892a3d4SEli Cohen counter_set_dealloc(ndev, mvq); 12831a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 12841a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 12851a86b377SEli Cohen cq_destroy(ndev, mvq->index); 12861a86b377SEli Cohen mvq->initialized = false; 12871a86b377SEli Cohen } 12881a86b377SEli Cohen 12891a86b377SEli Cohen static int create_rqt(struct mlx5_vdpa_net *ndev) 12901a86b377SEli Cohen { 12911a86b377SEli Cohen __be32 *list; 12921a86b377SEli Cohen void *rqtc; 12931a86b377SEli Cohen int inlen; 12941a86b377SEli Cohen void *in; 12951a86b377SEli Cohen int i, j; 12961a86b377SEli Cohen int err; 12971a86b377SEli Cohen 1298acde3929SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + ndev->rqt_size * MLX5_ST_SZ_BYTES(rq_num); 12991a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 13001a86b377SEli Cohen if (!in) 13011a86b377SEli Cohen return -ENOMEM; 13021a86b377SEli Cohen 13031a86b377SEli Cohen MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); 13041a86b377SEli Cohen rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 13051a86b377SEli Cohen 13061a86b377SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 1307acde3929SEli Cohen MLX5_SET(rqtc, rqtc, rqt_max_size, ndev->rqt_size); 13081a86b377SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1309acde3929SEli Cohen for (i = 0, j = 0; i < ndev->rqt_size; i++, j += 2) 1310acde3929SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j % ndev->cur_num_vqs].virtq_id); 13111a86b377SEli Cohen 1312acde3929SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, ndev->rqt_size); 13131a86b377SEli Cohen err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); 13141a86b377SEli Cohen kfree(in); 13151a86b377SEli Cohen if (err) 13161a86b377SEli Cohen return err; 13171a86b377SEli Cohen 13181a86b377SEli Cohen return 0; 13191a86b377SEli Cohen } 13201a86b377SEli Cohen 132152893733SEli Cohen #define MLX5_MODIFY_RQT_NUM_RQS ((u64)1) 132252893733SEli Cohen 132352893733SEli Cohen static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) 132452893733SEli Cohen { 132552893733SEli Cohen __be32 *list; 132652893733SEli Cohen void *rqtc; 132752893733SEli Cohen int inlen; 132852893733SEli Cohen void *in; 132952893733SEli Cohen int i, j; 133052893733SEli Cohen int err; 133152893733SEli Cohen 1332acde3929SEli Cohen inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + ndev->rqt_size * MLX5_ST_SZ_BYTES(rq_num); 133352893733SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 133452893733SEli Cohen if (!in) 133552893733SEli Cohen return -ENOMEM; 133652893733SEli Cohen 133752893733SEli Cohen MLX5_SET(modify_rqt_in, in, uid, ndev->mvdev.res.uid); 133852893733SEli Cohen MLX5_SET64(modify_rqt_in, in, bitmask, MLX5_MODIFY_RQT_NUM_RQS); 133952893733SEli Cohen rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 134052893733SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 134152893733SEli Cohen 134252893733SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1343acde3929SEli Cohen for (i = 0, j = 0; i < ndev->rqt_size; i++, j += 2) 1344a7f46ba4SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j % num].virtq_id); 134552893733SEli Cohen 1346acde3929SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, ndev->rqt_size); 134752893733SEli Cohen err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn); 134852893733SEli Cohen kfree(in); 134952893733SEli Cohen if (err) 135052893733SEli Cohen return err; 135152893733SEli Cohen 135252893733SEli Cohen return 0; 135352893733SEli Cohen } 135452893733SEli Cohen 13551a86b377SEli Cohen static void destroy_rqt(struct mlx5_vdpa_net *ndev) 13561a86b377SEli Cohen { 13571a86b377SEli Cohen mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); 13581a86b377SEli Cohen } 13591a86b377SEli Cohen 13601a86b377SEli Cohen static int create_tir(struct mlx5_vdpa_net *ndev) 13611a86b377SEli Cohen { 13621a86b377SEli Cohen #define HASH_IP_L4PORTS \ 13631a86b377SEli Cohen (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \ 13641a86b377SEli Cohen MLX5_HASH_FIELD_SEL_L4_DPORT) 13651a86b377SEli Cohen static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 13661a86b377SEli Cohen 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 13671a86b377SEli Cohen 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 13681a86b377SEli Cohen 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 13691a86b377SEli Cohen 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a }; 13701a86b377SEli Cohen void *rss_key; 13711a86b377SEli Cohen void *outer; 13721a86b377SEli Cohen void *tirc; 13731a86b377SEli Cohen void *in; 13741a86b377SEli Cohen int err; 13751a86b377SEli Cohen 13761a86b377SEli Cohen in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL); 13771a86b377SEli Cohen if (!in) 13781a86b377SEli Cohen return -ENOMEM; 13791a86b377SEli Cohen 13801a86b377SEli Cohen MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); 13811a86b377SEli Cohen tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 13821a86b377SEli Cohen MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 13831a86b377SEli Cohen 13841a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 13851a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ); 13861a86b377SEli Cohen rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 13871a86b377SEli Cohen memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key)); 13881a86b377SEli Cohen 13891a86b377SEli Cohen outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 13901a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); 13911a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); 13921a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS); 13931a86b377SEli Cohen 13941a86b377SEli Cohen MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); 13951a86b377SEli Cohen MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); 13961a86b377SEli Cohen 13971a86b377SEli Cohen err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); 13981a86b377SEli Cohen kfree(in); 13991a86b377SEli Cohen return err; 14001a86b377SEli Cohen } 14011a86b377SEli Cohen 14021a86b377SEli Cohen static void destroy_tir(struct mlx5_vdpa_net *ndev) 14031a86b377SEli Cohen { 14041a86b377SEli Cohen mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); 14051a86b377SEli Cohen } 14061a86b377SEli Cohen 14071a86b377SEli Cohen static int add_fwd_to_tir(struct mlx5_vdpa_net *ndev) 14081a86b377SEli Cohen { 14091a86b377SEli Cohen struct mlx5_flow_destination dest[2] = {}; 14101a86b377SEli Cohen struct mlx5_flow_table_attr ft_attr = {}; 14111a86b377SEli Cohen struct mlx5_flow_act flow_act = {}; 14121a86b377SEli Cohen struct mlx5_flow_namespace *ns; 1413540061acSEli Cohen struct mlx5_flow_spec *spec; 1414540061acSEli Cohen void *headers_c; 1415540061acSEli Cohen void *headers_v; 1416540061acSEli Cohen u8 *dmac_c; 1417540061acSEli Cohen u8 *dmac_v; 14181a86b377SEli Cohen int err; 14191a86b377SEli Cohen 1420540061acSEli Cohen spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1421540061acSEli Cohen if (!spec) 1422540061acSEli Cohen return -ENOMEM; 1423540061acSEli Cohen 1424540061acSEli Cohen spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1425540061acSEli Cohen ft_attr.max_fte = 2; 1426540061acSEli Cohen ft_attr.autogroup.max_num_groups = 2; 14271a86b377SEli Cohen 14281a86b377SEli Cohen ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); 14291a86b377SEli Cohen if (!ns) { 1430540061acSEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to get flow namespace\n"); 1431540061acSEli Cohen err = -EOPNOTSUPP; 1432540061acSEli Cohen goto err_ns; 14331a86b377SEli Cohen } 14341a86b377SEli Cohen 14351a86b377SEli Cohen ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 1436540061acSEli Cohen if (IS_ERR(ndev->rxft)) { 1437540061acSEli Cohen err = PTR_ERR(ndev->rxft); 1438540061acSEli Cohen goto err_ns; 1439540061acSEli Cohen } 14401a86b377SEli Cohen 14411a86b377SEli Cohen ndev->rx_counter = mlx5_fc_create(ndev->mvdev.mdev, false); 14421a86b377SEli Cohen if (IS_ERR(ndev->rx_counter)) { 14431a86b377SEli Cohen err = PTR_ERR(ndev->rx_counter); 14441a86b377SEli Cohen goto err_fc; 14451a86b377SEli Cohen } 14461a86b377SEli Cohen 1447540061acSEli Cohen headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers); 1448540061acSEli Cohen dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, outer_headers.dmac_47_16); 1449540061acSEli Cohen memset(dmac_c, 0xff, ETH_ALEN); 1450540061acSEli Cohen headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); 1451540061acSEli Cohen dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16); 1452540061acSEli Cohen ether_addr_copy(dmac_v, ndev->config.mac); 1453540061acSEli Cohen 14541a86b377SEli Cohen flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_COUNT; 14551a86b377SEli Cohen dest[0].type = MLX5_FLOW_DESTINATION_TYPE_TIR; 14561a86b377SEli Cohen dest[0].tir_num = ndev->res.tirn; 14571a86b377SEli Cohen dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 14581a86b377SEli Cohen dest[1].counter_id = mlx5_fc_id(ndev->rx_counter); 1459540061acSEli Cohen ndev->rx_rule_ucast = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dest, 2); 1460540061acSEli Cohen 1461540061acSEli Cohen if (IS_ERR(ndev->rx_rule_ucast)) { 1462540061acSEli Cohen err = PTR_ERR(ndev->rx_rule_ucast); 1463540061acSEli Cohen ndev->rx_rule_ucast = NULL; 1464540061acSEli Cohen goto err_rule_ucast; 14651a86b377SEli Cohen } 14661a86b377SEli Cohen 1467540061acSEli Cohen memset(dmac_c, 0, ETH_ALEN); 1468540061acSEli Cohen memset(dmac_v, 0, ETH_ALEN); 1469540061acSEli Cohen dmac_c[0] = 1; 1470540061acSEli Cohen dmac_v[0] = 1; 1471540061acSEli Cohen flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1472540061acSEli Cohen ndev->rx_rule_mcast = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dest, 1); 1473540061acSEli Cohen if (IS_ERR(ndev->rx_rule_mcast)) { 1474540061acSEli Cohen err = PTR_ERR(ndev->rx_rule_mcast); 1475540061acSEli Cohen ndev->rx_rule_mcast = NULL; 1476540061acSEli Cohen goto err_rule_mcast; 1477540061acSEli Cohen } 1478540061acSEli Cohen 1479540061acSEli Cohen kvfree(spec); 14801a86b377SEli Cohen return 0; 14811a86b377SEli Cohen 1482540061acSEli Cohen err_rule_mcast: 1483540061acSEli Cohen mlx5_del_flow_rules(ndev->rx_rule_ucast); 1484540061acSEli Cohen ndev->rx_rule_ucast = NULL; 1485540061acSEli Cohen err_rule_ucast: 14861a86b377SEli Cohen mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter); 14871a86b377SEli Cohen err_fc: 14881a86b377SEli Cohen mlx5_destroy_flow_table(ndev->rxft); 1489540061acSEli Cohen err_ns: 1490540061acSEli Cohen kvfree(spec); 14911a86b377SEli Cohen return err; 14921a86b377SEli Cohen } 14931a86b377SEli Cohen 14941a86b377SEli Cohen static void remove_fwd_to_tir(struct mlx5_vdpa_net *ndev) 14951a86b377SEli Cohen { 1496540061acSEli Cohen if (!ndev->rx_rule_ucast) 14971a86b377SEli Cohen return; 14981a86b377SEli Cohen 1499540061acSEli Cohen mlx5_del_flow_rules(ndev->rx_rule_mcast); 1500540061acSEli Cohen ndev->rx_rule_mcast = NULL; 1501540061acSEli Cohen mlx5_del_flow_rules(ndev->rx_rule_ucast); 1502540061acSEli Cohen ndev->rx_rule_ucast = NULL; 15031a86b377SEli Cohen mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter); 15041a86b377SEli Cohen mlx5_destroy_flow_table(ndev->rxft); 15051a86b377SEli Cohen } 15061a86b377SEli Cohen 15075262912eSEli Cohen static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) 15085262912eSEli Cohen { 15095262912eSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 15105262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 15115262912eSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 15125262912eSEli Cohen struct mlx5_core_dev *pfmdev; 15135262912eSEli Cohen size_t read; 1514f1781bedSMichael Qiu u8 mac[ETH_ALEN], mac_back[ETH_ALEN]; 15155262912eSEli Cohen 15165262912eSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 15175262912eSEli Cohen switch (cmd) { 15185262912eSEli Cohen case VIRTIO_NET_CTRL_MAC_ADDR_SET: 15195262912eSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)mac, ETH_ALEN); 15205262912eSEli Cohen if (read != ETH_ALEN) 15215262912eSEli Cohen break; 15225262912eSEli Cohen 15235262912eSEli Cohen if (!memcmp(ndev->config.mac, mac, 6)) { 15245262912eSEli Cohen status = VIRTIO_NET_OK; 15255262912eSEli Cohen break; 15265262912eSEli Cohen } 15275262912eSEli Cohen 1528f1781bedSMichael Qiu if (is_zero_ether_addr(mac)) 1529f1781bedSMichael Qiu break; 1530f1781bedSMichael Qiu 15315262912eSEli Cohen if (!is_zero_ether_addr(ndev->config.mac)) { 15325262912eSEli Cohen if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 15335262912eSEli Cohen mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n", 15345262912eSEli Cohen ndev->config.mac); 15355262912eSEli Cohen break; 15365262912eSEli Cohen } 15375262912eSEli Cohen } 15385262912eSEli Cohen 15395262912eSEli Cohen if (mlx5_mpfs_add_mac(pfmdev, mac)) { 15405262912eSEli Cohen mlx5_vdpa_warn(mvdev, "failed to insert new MAC %pM into MPFS table\n", 15415262912eSEli Cohen mac); 15425262912eSEli Cohen break; 15435262912eSEli Cohen } 15445262912eSEli Cohen 1545f1781bedSMichael Qiu /* backup the original mac address so that if failed to add the forward rules 1546f1781bedSMichael Qiu * we could restore it 1547f1781bedSMichael Qiu */ 1548f1781bedSMichael Qiu memcpy(mac_back, ndev->config.mac, ETH_ALEN); 1549f1781bedSMichael Qiu 15505262912eSEli Cohen memcpy(ndev->config.mac, mac, ETH_ALEN); 1551f1781bedSMichael Qiu 1552f1781bedSMichael Qiu /* Need recreate the flow table entry, so that the packet could forward back 1553f1781bedSMichael Qiu */ 1554f1781bedSMichael Qiu remove_fwd_to_tir(ndev); 1555f1781bedSMichael Qiu 1556f1781bedSMichael Qiu if (add_fwd_to_tir(ndev)) { 1557f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "failed to insert forward rules, try to restore\n"); 1558f1781bedSMichael Qiu 1559f1781bedSMichael Qiu /* Although it hardly run here, we still need double check */ 1560f1781bedSMichael Qiu if (is_zero_ether_addr(mac_back)) { 1561f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore mac failed: Original MAC is zero\n"); 1562f1781bedSMichael Qiu break; 1563f1781bedSMichael Qiu } 1564f1781bedSMichael Qiu 1565f1781bedSMichael Qiu /* Try to restore original mac address to MFPS table, and try to restore 1566f1781bedSMichael Qiu * the forward rule entry. 1567f1781bedSMichael Qiu */ 1568f1781bedSMichael Qiu if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 1569f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore mac failed: delete MAC %pM from MPFS table failed\n", 1570f1781bedSMichael Qiu ndev->config.mac); 1571f1781bedSMichael Qiu } 1572f1781bedSMichael Qiu 1573f1781bedSMichael Qiu if (mlx5_mpfs_add_mac(pfmdev, mac_back)) { 1574f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore mac failed: insert old MAC %pM into MPFS table failed\n", 1575f1781bedSMichael Qiu mac_back); 1576f1781bedSMichael Qiu } 1577f1781bedSMichael Qiu 1578f1781bedSMichael Qiu memcpy(ndev->config.mac, mac_back, ETH_ALEN); 1579f1781bedSMichael Qiu 1580f1781bedSMichael Qiu if (add_fwd_to_tir(ndev)) 1581f1781bedSMichael Qiu mlx5_vdpa_warn(mvdev, "restore forward rules failed: insert forward rules failed\n"); 1582f1781bedSMichael Qiu 1583f1781bedSMichael Qiu break; 1584f1781bedSMichael Qiu } 1585f1781bedSMichael Qiu 15865262912eSEli Cohen status = VIRTIO_NET_OK; 15875262912eSEli Cohen break; 15885262912eSEli Cohen 15895262912eSEli Cohen default: 15905262912eSEli Cohen break; 15915262912eSEli Cohen } 15925262912eSEli Cohen 15935262912eSEli Cohen return status; 15945262912eSEli Cohen } 15955262912eSEli Cohen 159652893733SEli Cohen static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps) 159752893733SEli Cohen { 159852893733SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 159952893733SEli Cohen int cur_qps = ndev->cur_num_vqs / 2; 160052893733SEli Cohen int err; 160152893733SEli Cohen int i; 160252893733SEli Cohen 160352893733SEli Cohen if (cur_qps > newqps) { 160452893733SEli Cohen err = modify_rqt(ndev, 2 * newqps); 160552893733SEli Cohen if (err) 160652893733SEli Cohen return err; 160752893733SEli Cohen 160852893733SEli Cohen for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) 160952893733SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 161052893733SEli Cohen 161152893733SEli Cohen ndev->cur_num_vqs = 2 * newqps; 161252893733SEli Cohen } else { 161352893733SEli Cohen ndev->cur_num_vqs = 2 * newqps; 161452893733SEli Cohen for (i = cur_qps * 2; i < 2 * newqps; i++) { 161552893733SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 161652893733SEli Cohen if (err) 161752893733SEli Cohen goto clean_added; 161852893733SEli Cohen } 161952893733SEli Cohen err = modify_rqt(ndev, 2 * newqps); 162052893733SEli Cohen if (err) 162152893733SEli Cohen goto clean_added; 162252893733SEli Cohen } 162352893733SEli Cohen return 0; 162452893733SEli Cohen 162552893733SEli Cohen clean_added: 162637e07e70SEli Cohen for (--i; i >= 2 * cur_qps; --i) 162752893733SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 162852893733SEli Cohen 162937e07e70SEli Cohen ndev->cur_num_vqs = 2 * cur_qps; 163037e07e70SEli Cohen 163152893733SEli Cohen return err; 163252893733SEli Cohen } 163352893733SEli Cohen 163452893733SEli Cohen static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd) 163552893733SEli Cohen { 163652893733SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 163752893733SEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 163852893733SEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 163952893733SEli Cohen struct virtio_net_ctrl_mq mq; 164052893733SEli Cohen size_t read; 164152893733SEli Cohen u16 newqps; 164252893733SEli Cohen 164352893733SEli Cohen switch (cmd) { 164452893733SEli Cohen case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: 1645ed0f849fSSi-Wei Liu /* This mq feature check aligns with pre-existing userspace 1646ed0f849fSSi-Wei Liu * implementation. 1647ed0f849fSSi-Wei Liu * 1648ed0f849fSSi-Wei Liu * Without it, an untrusted driver could fake a multiqueue config 1649ed0f849fSSi-Wei Liu * request down to a non-mq device that may cause kernel to 1650ed0f849fSSi-Wei Liu * panic due to uninitialized resources for extra vqs. Even with 1651ed0f849fSSi-Wei Liu * a well behaving guest driver, it is not expected to allow 1652ed0f849fSSi-Wei Liu * changing the number of vqs on a non-mq device. 1653ed0f849fSSi-Wei Liu */ 1654ed0f849fSSi-Wei Liu if (!MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) 1655ed0f849fSSi-Wei Liu break; 1656ed0f849fSSi-Wei Liu 165752893733SEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)&mq, sizeof(mq)); 165852893733SEli Cohen if (read != sizeof(mq)) 165952893733SEli Cohen break; 166052893733SEli Cohen 166152893733SEli Cohen newqps = mlx5vdpa16_to_cpu(mvdev, mq.virtqueue_pairs); 1662ed0f849fSSi-Wei Liu if (newqps < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 1663acde3929SEli Cohen newqps > ndev->rqt_size) 1664ed0f849fSSi-Wei Liu break; 1665ed0f849fSSi-Wei Liu 166652893733SEli Cohen if (ndev->cur_num_vqs == 2 * newqps) { 166752893733SEli Cohen status = VIRTIO_NET_OK; 166852893733SEli Cohen break; 166952893733SEli Cohen } 167052893733SEli Cohen 167152893733SEli Cohen if (!change_num_qps(mvdev, newqps)) 167252893733SEli Cohen status = VIRTIO_NET_OK; 167352893733SEli Cohen 167452893733SEli Cohen break; 167552893733SEli Cohen default: 167652893733SEli Cohen break; 167752893733SEli Cohen } 167852893733SEli Cohen 167952893733SEli Cohen return status; 168052893733SEli Cohen } 168152893733SEli Cohen 16825262912eSEli Cohen static void mlx5_cvq_kick_handler(struct work_struct *work) 16835262912eSEli Cohen { 16845262912eSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 16855262912eSEli Cohen struct virtio_net_ctrl_hdr ctrl; 1686218bdd20SEli Cohen struct mlx5_vdpa_wq_ent *wqent; 16875262912eSEli Cohen struct mlx5_vdpa_dev *mvdev; 16885262912eSEli Cohen struct mlx5_control_vq *cvq; 16895262912eSEli Cohen struct mlx5_vdpa_net *ndev; 16905262912eSEli Cohen size_t read, write; 16915262912eSEli Cohen int err; 16925262912eSEli Cohen 1693218bdd20SEli Cohen wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 16945262912eSEli Cohen mvdev = wqent->mvdev; 16955262912eSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 16965262912eSEli Cohen cvq = &mvdev->cvq; 16971c80cf03SJason Wang 1698759ae7f9SEli Cohen down_write(&ndev->reslock); 16991c80cf03SJason Wang 17001c80cf03SJason Wang if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 17011c80cf03SJason Wang goto out; 17021c80cf03SJason Wang 17035262912eSEli Cohen if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 17041c80cf03SJason Wang goto out; 17055262912eSEli Cohen 17065262912eSEli Cohen if (!cvq->ready) 17071c80cf03SJason Wang goto out; 17085262912eSEli Cohen 17095262912eSEli Cohen while (true) { 17105262912eSEli Cohen err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head, 17115262912eSEli Cohen GFP_ATOMIC); 17125262912eSEli Cohen if (err <= 0) 17135262912eSEli Cohen break; 17145262912eSEli Cohen 17155262912eSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &ctrl, sizeof(ctrl)); 17165262912eSEli Cohen if (read != sizeof(ctrl)) 17175262912eSEli Cohen break; 17185262912eSEli Cohen 17191892a3d4SEli Cohen cvq->received_desc++; 17205262912eSEli Cohen switch (ctrl.class) { 17215262912eSEli Cohen case VIRTIO_NET_CTRL_MAC: 17225262912eSEli Cohen status = handle_ctrl_mac(mvdev, ctrl.cmd); 17235262912eSEli Cohen break; 172452893733SEli Cohen case VIRTIO_NET_CTRL_MQ: 172552893733SEli Cohen status = handle_ctrl_mq(mvdev, ctrl.cmd); 172652893733SEli Cohen break; 17275262912eSEli Cohen 17285262912eSEli Cohen default: 17295262912eSEli Cohen break; 17305262912eSEli Cohen } 17315262912eSEli Cohen 17325262912eSEli Cohen /* Make sure data is written before advancing index */ 17335262912eSEli Cohen smp_wmb(); 17345262912eSEli Cohen 17355262912eSEli Cohen write = vringh_iov_push_iotlb(&cvq->vring, &cvq->wiov, &status, sizeof(status)); 17365262912eSEli Cohen vringh_complete_iotlb(&cvq->vring, cvq->head, write); 17375262912eSEli Cohen vringh_kiov_cleanup(&cvq->riov); 17385262912eSEli Cohen vringh_kiov_cleanup(&cvq->wiov); 17395262912eSEli Cohen 17405262912eSEli Cohen if (vringh_need_notify_iotlb(&cvq->vring)) 17415262912eSEli Cohen vringh_notify(&cvq->vring); 174255ebf0d6SJason Wang 17431892a3d4SEli Cohen cvq->completed_desc++; 174455ebf0d6SJason Wang queue_work(mvdev->wq, &wqent->work); 174555ebf0d6SJason Wang break; 17465262912eSEli Cohen } 17471c80cf03SJason Wang 17481c80cf03SJason Wang out: 1749759ae7f9SEli Cohen up_write(&ndev->reslock); 17505262912eSEli Cohen } 17515262912eSEli Cohen 17521a86b377SEli Cohen static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) 17531a86b377SEli Cohen { 17541a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17551a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17565262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 17571a86b377SEli Cohen 1758e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1759e4fc6650SEli Cohen return; 1760e4fc6650SEli Cohen 17615262912eSEli Cohen if (unlikely(is_ctrl_vq_idx(mvdev, idx))) { 1762ad6dc1daSEli Cohen if (!mvdev->wq || !mvdev->cvq.ready) 17635262912eSEli Cohen return; 17645262912eSEli Cohen 176555ebf0d6SJason Wang queue_work(mvdev->wq, &ndev->cvq_ent.work); 17665262912eSEli Cohen return; 17675262912eSEli Cohen } 17685262912eSEli Cohen 17695262912eSEli Cohen mvq = &ndev->vqs[idx]; 17701a86b377SEli Cohen if (unlikely(!mvq->ready)) 17711a86b377SEli Cohen return; 17721a86b377SEli Cohen 17731a86b377SEli Cohen iowrite16(idx, ndev->mvdev.res.kick_addr); 17741a86b377SEli Cohen } 17751a86b377SEli Cohen 17761a86b377SEli Cohen static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 17771a86b377SEli Cohen u64 driver_area, u64 device_area) 17781a86b377SEli Cohen { 17791a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17801a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17815262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 17821a86b377SEli Cohen 1783e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1784e4fc6650SEli Cohen return -EINVAL; 1785e4fc6650SEli Cohen 17865262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 17875262912eSEli Cohen mvdev->cvq.desc_addr = desc_area; 17885262912eSEli Cohen mvdev->cvq.device_addr = device_area; 17895262912eSEli Cohen mvdev->cvq.driver_addr = driver_area; 17905262912eSEli Cohen return 0; 17915262912eSEli Cohen } 17925262912eSEli Cohen 17935262912eSEli Cohen mvq = &ndev->vqs[idx]; 17941a86b377SEli Cohen mvq->desc_addr = desc_area; 17951a86b377SEli Cohen mvq->device_addr = device_area; 17961a86b377SEli Cohen mvq->driver_addr = driver_area; 17971a86b377SEli Cohen return 0; 17981a86b377SEli Cohen } 17991a86b377SEli Cohen 18001a86b377SEli Cohen static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 18011a86b377SEli Cohen { 18021a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 18031a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 18041a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 18051a86b377SEli Cohen 18065262912eSEli Cohen if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 1807e4fc6650SEli Cohen return; 1808e4fc6650SEli Cohen 18091a86b377SEli Cohen mvq = &ndev->vqs[idx]; 18101a86b377SEli Cohen mvq->num_ent = num; 18111a86b377SEli Cohen } 18121a86b377SEli Cohen 18131a86b377SEli Cohen static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 18141a86b377SEli Cohen { 18151a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 18161a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 18171a86b377SEli Cohen 1818db296d25SEli Cohen ndev->event_cbs[idx] = *cb; 18191a86b377SEli Cohen } 18201a86b377SEli Cohen 18215262912eSEli Cohen static void mlx5_cvq_notify(struct vringh *vring) 18225262912eSEli Cohen { 18235262912eSEli Cohen struct mlx5_control_vq *cvq = container_of(vring, struct mlx5_control_vq, vring); 18245262912eSEli Cohen 18255262912eSEli Cohen if (!cvq->event_cb.callback) 18265262912eSEli Cohen return; 18275262912eSEli Cohen 18285262912eSEli Cohen cvq->event_cb.callback(cvq->event_cb.private); 18295262912eSEli Cohen } 18305262912eSEli Cohen 18315262912eSEli Cohen static void set_cvq_ready(struct mlx5_vdpa_dev *mvdev, bool ready) 18325262912eSEli Cohen { 18335262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 18345262912eSEli Cohen 18355262912eSEli Cohen cvq->ready = ready; 18365262912eSEli Cohen if (!ready) 18375262912eSEli Cohen return; 18385262912eSEli Cohen 18395262912eSEli Cohen cvq->vring.notify = mlx5_cvq_notify; 18401a86b377SEli Cohen } 18411a86b377SEli Cohen 18421a86b377SEli Cohen static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 18431a86b377SEli Cohen { 18441a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 18451a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 18465262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 18471a86b377SEli Cohen 1848759be899SEli Cohen if (!mvdev->actual_features) 1849759be899SEli Cohen return; 1850759be899SEli Cohen 1851e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1852e4fc6650SEli Cohen return; 1853e4fc6650SEli Cohen 18545262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 18555262912eSEli Cohen set_cvq_ready(mvdev, ready); 18565262912eSEli Cohen return; 18575262912eSEli Cohen } 18585262912eSEli Cohen 18595262912eSEli Cohen mvq = &ndev->vqs[idx]; 18601a86b377SEli Cohen if (!ready) 18611a86b377SEli Cohen suspend_vq(ndev, mvq); 18621a86b377SEli Cohen 18631a86b377SEli Cohen mvq->ready = ready; 18641a86b377SEli Cohen } 18651a86b377SEli Cohen 18661a86b377SEli Cohen static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx) 18671a86b377SEli Cohen { 18681a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 18691a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 18701a86b377SEli Cohen 1871e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1872e4fc6650SEli Cohen return false; 1873e4fc6650SEli Cohen 18745262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) 18755262912eSEli Cohen return mvdev->cvq.ready; 18765262912eSEli Cohen 18775262912eSEli Cohen return ndev->vqs[idx].ready; 18781a86b377SEli Cohen } 18791a86b377SEli Cohen 18801a86b377SEli Cohen static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx, 18811a86b377SEli Cohen const struct vdpa_vq_state *state) 18821a86b377SEli Cohen { 18831a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 18841a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 18855262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 18861a86b377SEli Cohen 1887e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1888e4fc6650SEli Cohen return -EINVAL; 1889e4fc6650SEli Cohen 18905262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 18915262912eSEli Cohen mvdev->cvq.vring.last_avail_idx = state->split.avail_index; 18925262912eSEli Cohen return 0; 18935262912eSEli Cohen } 18945262912eSEli Cohen 18955262912eSEli Cohen mvq = &ndev->vqs[idx]; 18961a86b377SEli Cohen if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) { 18971a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "can't modify available index\n"); 18981a86b377SEli Cohen return -EINVAL; 18991a86b377SEli Cohen } 19001a86b377SEli Cohen 1901530a5678SJason Wang mvq->used_idx = state->split.avail_index; 1902530a5678SJason Wang mvq->avail_idx = state->split.avail_index; 19031a86b377SEli Cohen return 0; 19041a86b377SEli Cohen } 19051a86b377SEli Cohen 19061a86b377SEli Cohen static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 19071a86b377SEli Cohen { 19081a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19091a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19105262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 19111a86b377SEli Cohen struct mlx5_virtq_attr attr; 19121a86b377SEli Cohen int err; 19131a86b377SEli Cohen 1914e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1915e4fc6650SEli Cohen return -EINVAL; 1916e4fc6650SEli Cohen 19175262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 19185262912eSEli Cohen state->split.avail_index = mvdev->cvq.vring.last_avail_idx; 19195262912eSEli Cohen return 0; 19205262912eSEli Cohen } 19215262912eSEli Cohen 19225262912eSEli Cohen mvq = &ndev->vqs[idx]; 19233176e974SSi-Wei Liu /* If the virtq object was destroyed, use the value saved at 19243176e974SSi-Wei Liu * the last minute of suspend_vq. This caters for userspace 19253176e974SSi-Wei Liu * that cares about emulating the index after vq is stopped. 19263176e974SSi-Wei Liu */ 19273176e974SSi-Wei Liu if (!mvq->initialized) { 1928bc04d93eSEli Cohen /* Firmware returns a wrong value for the available index. 1929bc04d93eSEli Cohen * Since both values should be identical, we take the value of 1930bc04d93eSEli Cohen * used_idx which is reported correctly. 1931bc04d93eSEli Cohen */ 1932530a5678SJason Wang state->split.avail_index = mvq->used_idx; 19333176e974SSi-Wei Liu return 0; 19343176e974SSi-Wei Liu } 19351a86b377SEli Cohen 19361a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 19371a86b377SEli Cohen if (err) { 19381a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n"); 19391a86b377SEli Cohen return err; 19401a86b377SEli Cohen } 1941530a5678SJason Wang state->split.avail_index = attr.used_index; 19421a86b377SEli Cohen return 0; 19431a86b377SEli Cohen } 19441a86b377SEli Cohen 19451a86b377SEli Cohen static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) 19461a86b377SEli Cohen { 19471a86b377SEli Cohen return PAGE_SIZE; 19481a86b377SEli Cohen } 19491a86b377SEli Cohen 1950*d4821902SGautam Dawar static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdpa, u16 idx) 1951*d4821902SGautam Dawar { 1952*d4821902SGautam Dawar return 0; 1953*d4821902SGautam Dawar } 1954*d4821902SGautam Dawar 19551a86b377SEli Cohen enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9, 19561a86b377SEli Cohen MLX5_VIRTIO_NET_F_CSUM = 1 << 10, 19571a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11, 19581a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12, 19591a86b377SEli Cohen }; 19601a86b377SEli Cohen 19611a86b377SEli Cohen static u64 mlx_to_vritio_features(u16 dev_features) 19621a86b377SEli Cohen { 19631a86b377SEli Cohen u64 result = 0; 19641a86b377SEli Cohen 19651a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM) 1966cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM); 19671a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_CSUM) 1968cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_CSUM); 19691a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6) 1970cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6); 19711a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4) 1972cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4); 19731a86b377SEli Cohen 19741a86b377SEli Cohen return result; 19751a86b377SEli Cohen } 19761a86b377SEli Cohen 197779de65edSEli Cohen static u64 get_supported_features(struct mlx5_core_dev *mdev) 197879de65edSEli Cohen { 197979de65edSEli Cohen u64 mlx_vdpa_features = 0; 198079de65edSEli Cohen u16 dev_features; 198179de65edSEli Cohen 198279de65edSEli Cohen dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mdev, device_features_bits_mask); 198379de65edSEli Cohen mlx_vdpa_features |= mlx_to_vritio_features(dev_features); 198479de65edSEli Cohen if (MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_version_1_0)) 198579de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_F_VERSION_1); 198679de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM); 198779de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VQ); 198879de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR); 198979de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MQ); 199079de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_STATUS); 199179de65edSEli Cohen mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MTU); 199279de65edSEli Cohen 199379de65edSEli Cohen return mlx_vdpa_features; 199479de65edSEli Cohen } 199579de65edSEli Cohen 1996a64917bcSEli Cohen static u64 mlx5_vdpa_get_device_features(struct vdpa_device *vdev) 19971a86b377SEli Cohen { 19981a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19991a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20005262912eSEli Cohen 20011a86b377SEli Cohen print_features(mvdev, ndev->mvdev.mlx_features, false); 20021a86b377SEli Cohen return ndev->mvdev.mlx_features; 20031a86b377SEli Cohen } 20041a86b377SEli Cohen 200530c22f38SSi-Wei Liu static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features) 20061a86b377SEli Cohen { 200730c22f38SSi-Wei Liu /* Minimum features to expect */ 2008cbb52359SNathan Chancellor if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) 20091a86b377SEli Cohen return -EOPNOTSUPP; 20101a86b377SEli Cohen 201130c22f38SSi-Wei Liu /* Double check features combination sent down by the driver. 201230c22f38SSi-Wei Liu * Fail invalid features due to absence of the depended feature. 201330c22f38SSi-Wei Liu * 201430c22f38SSi-Wei Liu * Per VIRTIO v1.1 specification, section 5.1.3.1 Feature bit 201530c22f38SSi-Wei Liu * requirements: "VIRTIO_NET_F_MQ Requires VIRTIO_NET_F_CTRL_VQ". 201630c22f38SSi-Wei Liu * By failing the invalid features sent down by untrusted drivers, 201730c22f38SSi-Wei Liu * we're assured the assumption made upon is_index_valid() and 201830c22f38SSi-Wei Liu * is_ctrl_vq_idx() will not be compromised. 201930c22f38SSi-Wei Liu */ 202030c22f38SSi-Wei Liu if ((features & (BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) == 202130c22f38SSi-Wei Liu BIT_ULL(VIRTIO_NET_F_MQ)) 202230c22f38SSi-Wei Liu return -EINVAL; 202330c22f38SSi-Wei Liu 20241a86b377SEli Cohen return 0; 20251a86b377SEli Cohen } 20261a86b377SEli Cohen 2027ae0428deSEli Cohen static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev) 20281a86b377SEli Cohen { 2029ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20305262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 20311a86b377SEli Cohen int err; 20321a86b377SEli Cohen int i; 20331a86b377SEli Cohen 2034acde3929SEli Cohen for (i = 0; i < mvdev->max_vqs; i++) { 20351a86b377SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 20361a86b377SEli Cohen if (err) 20371a86b377SEli Cohen goto err_vq; 20381a86b377SEli Cohen } 20391a86b377SEli Cohen 20405262912eSEli Cohen if (mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) { 20415262912eSEli Cohen err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features, 20425262912eSEli Cohen MLX5_CVQ_MAX_ENT, false, 20435262912eSEli Cohen (struct vring_desc *)(uintptr_t)cvq->desc_addr, 20445262912eSEli Cohen (struct vring_avail *)(uintptr_t)cvq->driver_addr, 20455262912eSEli Cohen (struct vring_used *)(uintptr_t)cvq->device_addr); 20465262912eSEli Cohen if (err) 20475262912eSEli Cohen goto err_vq; 20485262912eSEli Cohen } 20495262912eSEli Cohen 20501a86b377SEli Cohen return 0; 20511a86b377SEli Cohen 20521a86b377SEli Cohen err_vq: 20531a86b377SEli Cohen for (--i; i >= 0; i--) 20541a86b377SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 20551a86b377SEli Cohen 20561a86b377SEli Cohen return err; 20571a86b377SEli Cohen } 20581a86b377SEli Cohen 20591a86b377SEli Cohen static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) 20601a86b377SEli Cohen { 20611a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 20621a86b377SEli Cohen int i; 20631a86b377SEli Cohen 20641a86b377SEli Cohen for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { 20651a86b377SEli Cohen mvq = &ndev->vqs[i]; 20661a86b377SEli Cohen if (!mvq->initialized) 20671a86b377SEli Cohen continue; 20681a86b377SEli Cohen 20691a86b377SEli Cohen teardown_vq(ndev, mvq); 20701a86b377SEli Cohen } 20711a86b377SEli Cohen } 20721a86b377SEli Cohen 2073e4fc6650SEli Cohen static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) 20748a7c3213SMichael S. Tsirkin { 2075e4fc6650SEli Cohen if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_CTRL_VQ)) { 2076e4fc6650SEli Cohen if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) { 2077e4fc6650SEli Cohen /* MQ supported. CVQ index is right above the last data virtqueue's */ 2078e4fc6650SEli Cohen mvdev->max_idx = mvdev->max_vqs; 2079e4fc6650SEli Cohen } else { 2080e4fc6650SEli Cohen /* Only CVQ supportted. data virtqueues occupy indices 0 and 1. 2081e4fc6650SEli Cohen * CVQ gets index 2 2082e4fc6650SEli Cohen */ 2083e4fc6650SEli Cohen mvdev->max_idx = 2; 20848a7c3213SMichael S. Tsirkin } 2085e4fc6650SEli Cohen } else { 2086e4fc6650SEli Cohen /* Two data virtqueues only: one for rx and one for tx */ 2087e4fc6650SEli Cohen mvdev->max_idx = 1; 2088e4fc6650SEli Cohen } 208936bdcf31SEli Cohen } 209036bdcf31SEli Cohen 2091a64917bcSEli Cohen static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) 20921a86b377SEli Cohen { 20931a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 20941a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20951a86b377SEli Cohen int err; 20961a86b377SEli Cohen 20971a86b377SEli Cohen print_features(mvdev, features, true); 20981a86b377SEli Cohen 209930c22f38SSi-Wei Liu err = verify_driver_features(mvdev, features); 21001a86b377SEli Cohen if (err) 21011a86b377SEli Cohen return err; 21021a86b377SEli Cohen 21031a86b377SEli Cohen ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; 2104b03fc43eSEli Cohen if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)) 2105acde3929SEli Cohen ndev->rqt_size = mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs); 2106b03fc43eSEli Cohen else 2107acde3929SEli Cohen ndev->rqt_size = 1; 2108acde3929SEli Cohen 2109acde3929SEli Cohen ndev->cur_num_vqs = 2 * ndev->rqt_size; 2110b03fc43eSEli Cohen 2111e4fc6650SEli Cohen update_cvq_info(mvdev); 21121a86b377SEli Cohen return err; 21131a86b377SEli Cohen } 21141a86b377SEli Cohen 21151a86b377SEli Cohen static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 21161a86b377SEli Cohen { 2117edf747afSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2118edf747afSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2119edf747afSEli Cohen 2120edf747afSEli Cohen ndev->config_cb = *cb; 21211a86b377SEli Cohen } 21221a86b377SEli Cohen 21231a86b377SEli Cohen #define MLX5_VDPA_MAX_VQ_ENTRIES 256 21241a86b377SEli Cohen static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev) 21251a86b377SEli Cohen { 21261a86b377SEli Cohen return MLX5_VDPA_MAX_VQ_ENTRIES; 21271a86b377SEli Cohen } 21281a86b377SEli Cohen 21291a86b377SEli Cohen static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev) 21301a86b377SEli Cohen { 21311a86b377SEli Cohen return VIRTIO_ID_NET; 21321a86b377SEli Cohen } 21331a86b377SEli Cohen 21341a86b377SEli Cohen static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev) 21351a86b377SEli Cohen { 21361a86b377SEli Cohen return PCI_VENDOR_ID_MELLANOX; 21371a86b377SEli Cohen } 21381a86b377SEli Cohen 21391a86b377SEli Cohen static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev) 21401a86b377SEli Cohen { 21411a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 21421a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 21431a86b377SEli Cohen 21441a86b377SEli Cohen print_status(mvdev, ndev->mvdev.status, false); 21451a86b377SEli Cohen return ndev->mvdev.status; 21461a86b377SEli Cohen } 21471a86b377SEli Cohen 21481a86b377SEli Cohen static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 21491a86b377SEli Cohen { 21501a86b377SEli Cohen struct mlx5_vq_restore_info *ri = &mvq->ri; 215152893733SEli Cohen struct mlx5_virtq_attr attr = {}; 21521a86b377SEli Cohen int err; 21531a86b377SEli Cohen 215452893733SEli Cohen if (mvq->initialized) { 21551a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 21561a86b377SEli Cohen if (err) 21571a86b377SEli Cohen return err; 215852893733SEli Cohen } 21591a86b377SEli Cohen 21601a86b377SEli Cohen ri->avail_index = attr.available_index; 2161b35ccebeSEli Cohen ri->used_index = attr.used_index; 21621a86b377SEli Cohen ri->ready = mvq->ready; 21631a86b377SEli Cohen ri->num_ent = mvq->num_ent; 21641a86b377SEli Cohen ri->desc_addr = mvq->desc_addr; 21651a86b377SEli Cohen ri->device_addr = mvq->device_addr; 21661a86b377SEli Cohen ri->driver_addr = mvq->driver_addr; 21671a86b377SEli Cohen ri->restore = true; 21681a86b377SEli Cohen return 0; 21691a86b377SEli Cohen } 21701a86b377SEli Cohen 21711a86b377SEli Cohen static int save_channels_info(struct mlx5_vdpa_net *ndev) 21721a86b377SEli Cohen { 21731a86b377SEli Cohen int i; 21741a86b377SEli Cohen 21751a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 21761a86b377SEli Cohen memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); 21771a86b377SEli Cohen save_channel_info(ndev, &ndev->vqs[i]); 21781a86b377SEli Cohen } 21791a86b377SEli Cohen return 0; 21801a86b377SEli Cohen } 21811a86b377SEli Cohen 21821a86b377SEli Cohen static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) 21831a86b377SEli Cohen { 21841a86b377SEli Cohen int i; 21851a86b377SEli Cohen 21861a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 21871a86b377SEli Cohen memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 21881a86b377SEli Cohen } 21891a86b377SEli Cohen 21901a86b377SEli Cohen static void restore_channels_info(struct mlx5_vdpa_net *ndev) 21911a86b377SEli Cohen { 21921a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 21931a86b377SEli Cohen struct mlx5_vq_restore_info *ri; 21941a86b377SEli Cohen int i; 21951a86b377SEli Cohen 21961a86b377SEli Cohen mlx5_clear_vqs(ndev); 21971a86b377SEli Cohen init_mvqs(ndev); 21981a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 21991a86b377SEli Cohen mvq = &ndev->vqs[i]; 22001a86b377SEli Cohen ri = &mvq->ri; 22011a86b377SEli Cohen if (!ri->restore) 22021a86b377SEli Cohen continue; 22031a86b377SEli Cohen 22041a86b377SEli Cohen mvq->avail_idx = ri->avail_index; 2205b35ccebeSEli Cohen mvq->used_idx = ri->used_index; 22061a86b377SEli Cohen mvq->ready = ri->ready; 22071a86b377SEli Cohen mvq->num_ent = ri->num_ent; 22081a86b377SEli Cohen mvq->desc_addr = ri->desc_addr; 22091a86b377SEli Cohen mvq->device_addr = ri->device_addr; 22101a86b377SEli Cohen mvq->driver_addr = ri->driver_addr; 22111a86b377SEli Cohen } 22121a86b377SEli Cohen } 22131a86b377SEli Cohen 2214ae0428deSEli Cohen static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) 22151a86b377SEli Cohen { 2216ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 22171a86b377SEli Cohen int err; 22181a86b377SEli Cohen 22191a86b377SEli Cohen suspend_vqs(ndev); 22201a86b377SEli Cohen err = save_channels_info(ndev); 22211a86b377SEli Cohen if (err) 22221a86b377SEli Cohen goto err_mr; 22231a86b377SEli Cohen 22241a86b377SEli Cohen teardown_driver(ndev); 2225ae0428deSEli Cohen mlx5_vdpa_destroy_mr(mvdev); 2226ae0428deSEli Cohen err = mlx5_vdpa_create_mr(mvdev, iotlb); 22271a86b377SEli Cohen if (err) 22281a86b377SEli Cohen goto err_mr; 22291a86b377SEli Cohen 2230ae0428deSEli Cohen if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 22311c80cf03SJason Wang goto err_mr; 22321897f0b6SEli Cohen 22331a86b377SEli Cohen restore_channels_info(ndev); 2234ae0428deSEli Cohen err = setup_driver(mvdev); 22351a86b377SEli Cohen if (err) 22361a86b377SEli Cohen goto err_setup; 22371a86b377SEli Cohen 22381a86b377SEli Cohen return 0; 22391a86b377SEli Cohen 22401a86b377SEli Cohen err_setup: 2241ae0428deSEli Cohen mlx5_vdpa_destroy_mr(mvdev); 22421a86b377SEli Cohen err_mr: 22431a86b377SEli Cohen return err; 22441a86b377SEli Cohen } 22451a86b377SEli Cohen 22461c80cf03SJason Wang /* reslock must be held for this function */ 2247ae0428deSEli Cohen static int setup_driver(struct mlx5_vdpa_dev *mvdev) 22481a86b377SEli Cohen { 2249ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 22501a86b377SEli Cohen int err; 22511a86b377SEli Cohen 2252759ae7f9SEli Cohen WARN_ON(!rwsem_is_locked(&ndev->reslock)); 22531c80cf03SJason Wang 22541a86b377SEli Cohen if (ndev->setup) { 2255ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n"); 22561a86b377SEli Cohen err = 0; 22571a86b377SEli Cohen goto out; 22581a86b377SEli Cohen } 2259ae0428deSEli Cohen err = setup_virtqueues(mvdev); 22601a86b377SEli Cohen if (err) { 2261ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "setup_virtqueues\n"); 22621a86b377SEli Cohen goto out; 22631a86b377SEli Cohen } 22641a86b377SEli Cohen 22651a86b377SEli Cohen err = create_rqt(ndev); 22661a86b377SEli Cohen if (err) { 2267ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "create_rqt\n"); 22681a86b377SEli Cohen goto err_rqt; 22691a86b377SEli Cohen } 22701a86b377SEli Cohen 22711a86b377SEli Cohen err = create_tir(ndev); 22721a86b377SEli Cohen if (err) { 2273ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "create_tir\n"); 22741a86b377SEli Cohen goto err_tir; 22751a86b377SEli Cohen } 22761a86b377SEli Cohen 22771a86b377SEli Cohen err = add_fwd_to_tir(ndev); 22781a86b377SEli Cohen if (err) { 2279ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "add_fwd_to_tir\n"); 22801a86b377SEli Cohen goto err_fwd; 22811a86b377SEli Cohen } 22821a86b377SEli Cohen ndev->setup = true; 22831a86b377SEli Cohen 22841a86b377SEli Cohen return 0; 22851a86b377SEli Cohen 22861a86b377SEli Cohen err_fwd: 22871a86b377SEli Cohen destroy_tir(ndev); 22881a86b377SEli Cohen err_tir: 22891a86b377SEli Cohen destroy_rqt(ndev); 22901a86b377SEli Cohen err_rqt: 22911a86b377SEli Cohen teardown_virtqueues(ndev); 22921a86b377SEli Cohen out: 22931a86b377SEli Cohen return err; 22941a86b377SEli Cohen } 22951a86b377SEli Cohen 22961c80cf03SJason Wang /* reslock must be held for this function */ 22971a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev) 22981a86b377SEli Cohen { 22991c80cf03SJason Wang 2300759ae7f9SEli Cohen WARN_ON(!rwsem_is_locked(&ndev->reslock)); 23011c80cf03SJason Wang 23021a86b377SEli Cohen if (!ndev->setup) 23031c80cf03SJason Wang return; 23041a86b377SEli Cohen 23051a86b377SEli Cohen remove_fwd_to_tir(ndev); 23061a86b377SEli Cohen destroy_tir(ndev); 23071a86b377SEli Cohen destroy_rqt(ndev); 23081a86b377SEli Cohen teardown_virtqueues(ndev); 23091a86b377SEli Cohen ndev->setup = false; 23101a86b377SEli Cohen } 23111a86b377SEli Cohen 2312e3aadf2eSEli Cohen static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) 2313e3aadf2eSEli Cohen { 2314e3aadf2eSEli Cohen int i; 2315e3aadf2eSEli Cohen 2316e3aadf2eSEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 2317e3aadf2eSEli Cohen ndev->vqs[i].ready = false; 2318ef12e4bfSEli Cohen 2319ef12e4bfSEli Cohen ndev->mvdev.cvq.ready = false; 2320e3aadf2eSEli Cohen } 2321e3aadf2eSEli Cohen 23221a86b377SEli Cohen static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) 23231a86b377SEli Cohen { 23241a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 23251a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 23261a86b377SEli Cohen int err; 23271a86b377SEli Cohen 23281a86b377SEli Cohen print_status(mvdev, status, true); 23291a86b377SEli Cohen 2330759ae7f9SEli Cohen down_write(&ndev->reslock); 23311c80cf03SJason Wang 23321a86b377SEli Cohen if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { 23331a86b377SEli Cohen if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 2334ae0428deSEli Cohen err = setup_driver(mvdev); 23351a86b377SEli Cohen if (err) { 23361a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); 23371a86b377SEli Cohen goto err_setup; 23381a86b377SEli Cohen } 23391a86b377SEli Cohen } else { 23401a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); 23411c80cf03SJason Wang goto err_clear; 23421a86b377SEli Cohen } 23431a86b377SEli Cohen } 23441a86b377SEli Cohen 23451a86b377SEli Cohen ndev->mvdev.status = status; 2346759ae7f9SEli Cohen up_write(&ndev->reslock); 23471a86b377SEli Cohen return; 23481a86b377SEli Cohen 23491a86b377SEli Cohen err_setup: 23501a86b377SEli Cohen mlx5_vdpa_destroy_mr(&ndev->mvdev); 23511a86b377SEli Cohen ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; 23521c80cf03SJason Wang err_clear: 2353759ae7f9SEli Cohen up_write(&ndev->reslock); 23541a86b377SEli Cohen } 23551a86b377SEli Cohen 23560686082dSXie Yongji static int mlx5_vdpa_reset(struct vdpa_device *vdev) 23570686082dSXie Yongji { 23580686082dSXie Yongji struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 23590686082dSXie Yongji struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 23600686082dSXie Yongji 23610686082dSXie Yongji print_status(mvdev, 0, true); 23620686082dSXie Yongji mlx5_vdpa_info(mvdev, "performing device reset\n"); 23631c80cf03SJason Wang 2364759ae7f9SEli Cohen down_write(&ndev->reslock); 23650686082dSXie Yongji teardown_driver(ndev); 23660686082dSXie Yongji clear_vqs_ready(ndev); 23670686082dSXie Yongji mlx5_vdpa_destroy_mr(&ndev->mvdev); 23680686082dSXie Yongji ndev->mvdev.status = 0; 2369b03fc43eSEli Cohen ndev->cur_num_vqs = 0; 23701892a3d4SEli Cohen ndev->mvdev.cvq.received_desc = 0; 23711892a3d4SEli Cohen ndev->mvdev.cvq.completed_desc = 0; 237275560522SEli Cohen memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1)); 23730686082dSXie Yongji ndev->mvdev.actual_features = 0; 23740686082dSXie Yongji ++mvdev->generation; 23750686082dSXie Yongji if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 23760686082dSXie Yongji if (mlx5_vdpa_create_mr(mvdev, NULL)) 23770686082dSXie Yongji mlx5_vdpa_warn(mvdev, "create MR failed\n"); 23780686082dSXie Yongji } 2379759ae7f9SEli Cohen up_write(&ndev->reslock); 23800686082dSXie Yongji 23810686082dSXie Yongji return 0; 23820686082dSXie Yongji } 23830686082dSXie Yongji 2384442706f9SStefano Garzarella static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev) 2385442706f9SStefano Garzarella { 2386442706f9SStefano Garzarella return sizeof(struct virtio_net_config); 2387442706f9SStefano Garzarella } 2388442706f9SStefano Garzarella 23891a86b377SEli Cohen static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, 23901a86b377SEli Cohen unsigned int len) 23911a86b377SEli Cohen { 23921a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 23931a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 23941a86b377SEli Cohen 2395dcfde163SStefano Garzarella if (offset + len <= sizeof(struct virtio_net_config)) 23962874211fSDan Carpenter memcpy(buf, (u8 *)&ndev->config + offset, len); 23971a86b377SEli Cohen } 23981a86b377SEli Cohen 23991a86b377SEli Cohen static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf, 24001a86b377SEli Cohen unsigned int len) 24011a86b377SEli Cohen { 24021a86b377SEli Cohen /* not supported */ 24031a86b377SEli Cohen } 24041a86b377SEli Cohen 24051a86b377SEli Cohen static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) 24061a86b377SEli Cohen { 24071a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 24081a86b377SEli Cohen 24091a86b377SEli Cohen return mvdev->generation; 24101a86b377SEli Cohen } 24111a86b377SEli Cohen 24121a86b377SEli Cohen static int mlx5_vdpa_set_map(struct vdpa_device *vdev, struct vhost_iotlb *iotlb) 24131a86b377SEli Cohen { 24141a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 24151c80cf03SJason Wang struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 24161a86b377SEli Cohen bool change_map; 24171a86b377SEli Cohen int err; 24181a86b377SEli Cohen 2419759ae7f9SEli Cohen down_write(&ndev->reslock); 24201c80cf03SJason Wang 24211a86b377SEli Cohen err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map); 24221a86b377SEli Cohen if (err) { 24231a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); 24241c80cf03SJason Wang goto err; 24251a86b377SEli Cohen } 24261a86b377SEli Cohen 24271a86b377SEli Cohen if (change_map) 24281c80cf03SJason Wang err = mlx5_vdpa_change_map(mvdev, iotlb); 24291a86b377SEli Cohen 24301c80cf03SJason Wang err: 2431759ae7f9SEli Cohen up_write(&ndev->reslock); 24321c80cf03SJason Wang return err; 24331a86b377SEli Cohen } 24341a86b377SEli Cohen 24351a86b377SEli Cohen static void mlx5_vdpa_free(struct vdpa_device *vdev) 24361a86b377SEli Cohen { 24371a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 24387c9f131fSEli Cohen struct mlx5_core_dev *pfmdev; 24391a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 24401a86b377SEli Cohen 24411a86b377SEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 24421a86b377SEli Cohen 24431a86b377SEli Cohen free_resources(ndev); 24446f5312f8SEli Cohen mlx5_vdpa_destroy_mr(mvdev); 24457c9f131fSEli Cohen if (!is_zero_ether_addr(ndev->config.mac)) { 24467c9f131fSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 24477c9f131fSEli Cohen mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); 24487c9f131fSEli Cohen } 24491a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 245075560522SEli Cohen kfree(ndev->event_cbs); 245175560522SEli Cohen kfree(ndev->vqs); 24521a86b377SEli Cohen } 24531a86b377SEli Cohen 24541a86b377SEli Cohen static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx) 24551a86b377SEli Cohen { 2456b57c46cbSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 24571a86b377SEli Cohen struct vdpa_notification_area ret = {}; 2458b57c46cbSEli Cohen struct mlx5_vdpa_net *ndev; 2459b57c46cbSEli Cohen phys_addr_t addr; 24601a86b377SEli Cohen 24615262912eSEli Cohen if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2462e4fc6650SEli Cohen return ret; 2463e4fc6650SEli Cohen 2464b57c46cbSEli Cohen /* If SF BAR size is smaller than PAGE_SIZE, do not use direct 2465b57c46cbSEli Cohen * notification to avoid the risk of mapping pages that contain BAR of more 2466b57c46cbSEli Cohen * than one SF 2467b57c46cbSEli Cohen */ 2468b57c46cbSEli Cohen if (MLX5_CAP_GEN(mvdev->mdev, log_min_sf_size) + 12 < PAGE_SHIFT) 2469b57c46cbSEli Cohen return ret; 2470b57c46cbSEli Cohen 2471b57c46cbSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 2472b57c46cbSEli Cohen addr = (phys_addr_t)ndev->mvdev.res.phys_kick_addr; 2473b57c46cbSEli Cohen ret.addr = addr; 2474b57c46cbSEli Cohen ret.size = PAGE_SIZE; 24751a86b377SEli Cohen return ret; 24761a86b377SEli Cohen } 24771a86b377SEli Cohen 24781a86b377SEli Cohen static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) 24791a86b377SEli Cohen { 24801a86b377SEli Cohen return -EOPNOTSUPP; 24811a86b377SEli Cohen } 24821a86b377SEli Cohen 2483a64917bcSEli Cohen static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) 2484a64917bcSEli Cohen { 2485a64917bcSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2486a64917bcSEli Cohen 2487a64917bcSEli Cohen return mvdev->actual_features; 2488a64917bcSEli Cohen } 2489a64917bcSEli Cohen 24901892a3d4SEli Cohen static int counter_set_query(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 24911892a3d4SEli Cohen u64 *received_desc, u64 *completed_desc) 24921892a3d4SEli Cohen { 24931892a3d4SEli Cohen u32 in[MLX5_ST_SZ_DW(query_virtio_q_counters_in)] = {}; 24941892a3d4SEli Cohen u32 out[MLX5_ST_SZ_DW(query_virtio_q_counters_out)] = {}; 24951892a3d4SEli Cohen void *cmd_hdr; 24961892a3d4SEli Cohen void *ctx; 24971892a3d4SEli Cohen int err; 24981892a3d4SEli Cohen 24991892a3d4SEli Cohen if (!counters_supported(&ndev->mvdev)) 25001892a3d4SEli Cohen return -EOPNOTSUPP; 25011892a3d4SEli Cohen 25021892a3d4SEli Cohen if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 25031892a3d4SEli Cohen return -EAGAIN; 25041892a3d4SEli Cohen 25051892a3d4SEli Cohen cmd_hdr = MLX5_ADDR_OF(query_virtio_q_counters_in, in, hdr); 25061892a3d4SEli Cohen 25071892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 25081892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 25091892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 25101892a3d4SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->counter_set_id); 25111892a3d4SEli Cohen 25121892a3d4SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 25131892a3d4SEli Cohen if (err) 25141892a3d4SEli Cohen return err; 25151892a3d4SEli Cohen 25161892a3d4SEli Cohen ctx = MLX5_ADDR_OF(query_virtio_q_counters_out, out, counters); 25171892a3d4SEli Cohen *received_desc = MLX5_GET64(virtio_q_counters, ctx, received_desc); 25181892a3d4SEli Cohen *completed_desc = MLX5_GET64(virtio_q_counters, ctx, completed_desc); 25191892a3d4SEli Cohen return 0; 25201892a3d4SEli Cohen } 25211892a3d4SEli Cohen 25221892a3d4SEli Cohen static int mlx5_vdpa_get_vendor_vq_stats(struct vdpa_device *vdev, u16 idx, 25231892a3d4SEli Cohen struct sk_buff *msg, 25241892a3d4SEli Cohen struct netlink_ext_ack *extack) 25251892a3d4SEli Cohen { 25261892a3d4SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 25271892a3d4SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 25281892a3d4SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 25291892a3d4SEli Cohen struct mlx5_control_vq *cvq; 25301892a3d4SEli Cohen u64 received_desc; 25311892a3d4SEli Cohen u64 completed_desc; 25321892a3d4SEli Cohen int err = 0; 25331892a3d4SEli Cohen 2534759ae7f9SEli Cohen down_read(&ndev->reslock); 25351892a3d4SEli Cohen if (!is_index_valid(mvdev, idx)) { 25361892a3d4SEli Cohen NL_SET_ERR_MSG_MOD(extack, "virtqueue index is not valid"); 25371892a3d4SEli Cohen err = -EINVAL; 25381892a3d4SEli Cohen goto out_err; 25391892a3d4SEli Cohen } 25401892a3d4SEli Cohen 25411892a3d4SEli Cohen if (idx == ctrl_vq_idx(mvdev)) { 25421892a3d4SEli Cohen cvq = &mvdev->cvq; 25431892a3d4SEli Cohen received_desc = cvq->received_desc; 25441892a3d4SEli Cohen completed_desc = cvq->completed_desc; 25451892a3d4SEli Cohen goto out; 25461892a3d4SEli Cohen } 25471892a3d4SEli Cohen 25481892a3d4SEli Cohen mvq = &ndev->vqs[idx]; 25491892a3d4SEli Cohen err = counter_set_query(ndev, mvq, &received_desc, &completed_desc); 25501892a3d4SEli Cohen if (err) { 25511892a3d4SEli Cohen NL_SET_ERR_MSG_MOD(extack, "failed to query hardware"); 25521892a3d4SEli Cohen goto out_err; 25531892a3d4SEli Cohen } 25541892a3d4SEli Cohen 25551892a3d4SEli Cohen out: 25561892a3d4SEli Cohen err = -EMSGSIZE; 25571892a3d4SEli Cohen if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "received_desc")) 25581892a3d4SEli Cohen goto out_err; 25591892a3d4SEli Cohen 25601892a3d4SEli Cohen if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, received_desc, 25611892a3d4SEli Cohen VDPA_ATTR_PAD)) 25621892a3d4SEli Cohen goto out_err; 25631892a3d4SEli Cohen 25641892a3d4SEli Cohen if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "completed_desc")) 25651892a3d4SEli Cohen goto out_err; 25661892a3d4SEli Cohen 25671892a3d4SEli Cohen if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, completed_desc, 25681892a3d4SEli Cohen VDPA_ATTR_PAD)) 25691892a3d4SEli Cohen goto out_err; 25701892a3d4SEli Cohen 25711892a3d4SEli Cohen err = 0; 25721892a3d4SEli Cohen out_err: 2573759ae7f9SEli Cohen up_read(&ndev->reslock); 25741892a3d4SEli Cohen return err; 25751892a3d4SEli Cohen } 25761892a3d4SEli Cohen 25771a86b377SEli Cohen static const struct vdpa_config_ops mlx5_vdpa_ops = { 25781a86b377SEli Cohen .set_vq_address = mlx5_vdpa_set_vq_address, 25791a86b377SEli Cohen .set_vq_num = mlx5_vdpa_set_vq_num, 25801a86b377SEli Cohen .kick_vq = mlx5_vdpa_kick_vq, 25811a86b377SEli Cohen .set_vq_cb = mlx5_vdpa_set_vq_cb, 25821a86b377SEli Cohen .set_vq_ready = mlx5_vdpa_set_vq_ready, 25831a86b377SEli Cohen .get_vq_ready = mlx5_vdpa_get_vq_ready, 25841a86b377SEli Cohen .set_vq_state = mlx5_vdpa_set_vq_state, 25851a86b377SEli Cohen .get_vq_state = mlx5_vdpa_get_vq_state, 25861892a3d4SEli Cohen .get_vendor_vq_stats = mlx5_vdpa_get_vendor_vq_stats, 25871a86b377SEli Cohen .get_vq_notification = mlx5_get_vq_notification, 25881a86b377SEli Cohen .get_vq_irq = mlx5_get_vq_irq, 25891a86b377SEli Cohen .get_vq_align = mlx5_vdpa_get_vq_align, 2590*d4821902SGautam Dawar .get_vq_group = mlx5_vdpa_get_vq_group, 2591a64917bcSEli Cohen .get_device_features = mlx5_vdpa_get_device_features, 2592a64917bcSEli Cohen .set_driver_features = mlx5_vdpa_set_driver_features, 2593a64917bcSEli Cohen .get_driver_features = mlx5_vdpa_get_driver_features, 25941a86b377SEli Cohen .set_config_cb = mlx5_vdpa_set_config_cb, 25951a86b377SEli Cohen .get_vq_num_max = mlx5_vdpa_get_vq_num_max, 25961a86b377SEli Cohen .get_device_id = mlx5_vdpa_get_device_id, 25971a86b377SEli Cohen .get_vendor_id = mlx5_vdpa_get_vendor_id, 25981a86b377SEli Cohen .get_status = mlx5_vdpa_get_status, 25991a86b377SEli Cohen .set_status = mlx5_vdpa_set_status, 26000686082dSXie Yongji .reset = mlx5_vdpa_reset, 2601442706f9SStefano Garzarella .get_config_size = mlx5_vdpa_get_config_size, 26021a86b377SEli Cohen .get_config = mlx5_vdpa_get_config, 26031a86b377SEli Cohen .set_config = mlx5_vdpa_set_config, 26041a86b377SEli Cohen .get_generation = mlx5_vdpa_get_generation, 26051a86b377SEli Cohen .set_map = mlx5_vdpa_set_map, 26061a86b377SEli Cohen .free = mlx5_vdpa_free, 26071a86b377SEli Cohen }; 26081a86b377SEli Cohen 2609d084d996SSi-Wei Liu static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) 2610d084d996SSi-Wei Liu { 2611d084d996SSi-Wei Liu u16 hw_mtu; 2612d084d996SSi-Wei Liu int err; 2613d084d996SSi-Wei Liu 2614d084d996SSi-Wei Liu err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2615d084d996SSi-Wei Liu if (err) 2616d084d996SSi-Wei Liu return err; 2617d084d996SSi-Wei Liu 2618d084d996SSi-Wei Liu *mtu = hw_mtu - MLX5V_ETH_HARD_MTU; 2619d084d996SSi-Wei Liu return 0; 2620d084d996SSi-Wei Liu } 2621d084d996SSi-Wei Liu 26221a86b377SEli Cohen static int alloc_resources(struct mlx5_vdpa_net *ndev) 26231a86b377SEli Cohen { 26241a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 26251a86b377SEli Cohen int err; 26261a86b377SEli Cohen 26271a86b377SEli Cohen if (res->valid) { 26281a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); 26291a86b377SEli Cohen return -EEXIST; 26301a86b377SEli Cohen } 26311a86b377SEli Cohen 26321a86b377SEli Cohen err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); 26331a86b377SEli Cohen if (err) 26341a86b377SEli Cohen return err; 26351a86b377SEli Cohen 26361a86b377SEli Cohen err = create_tis(ndev); 26371a86b377SEli Cohen if (err) 26381a86b377SEli Cohen goto err_tis; 26391a86b377SEli Cohen 26401a86b377SEli Cohen res->valid = true; 26411a86b377SEli Cohen 26421a86b377SEli Cohen return 0; 26431a86b377SEli Cohen 26441a86b377SEli Cohen err_tis: 26451a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 26461a86b377SEli Cohen return err; 26471a86b377SEli Cohen } 26481a86b377SEli Cohen 26491a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev) 26501a86b377SEli Cohen { 26511a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 26521a86b377SEli Cohen 26531a86b377SEli Cohen if (!res->valid) 26541a86b377SEli Cohen return; 26551a86b377SEli Cohen 26561a86b377SEli Cohen destroy_tis(ndev); 26571a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 26581a86b377SEli Cohen res->valid = false; 26591a86b377SEli Cohen } 26601a86b377SEli Cohen 26611a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev) 26621a86b377SEli Cohen { 26631a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 26641a86b377SEli Cohen int i; 26651a86b377SEli Cohen 2666acde3929SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; ++i) { 26671a86b377SEli Cohen mvq = &ndev->vqs[i]; 26681a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 26691a86b377SEli Cohen mvq->index = i; 26701a86b377SEli Cohen mvq->ndev = ndev; 26711a86b377SEli Cohen mvq->fwqp.fw = true; 26721a86b377SEli Cohen } 26731a86b377SEli Cohen for (; i < ndev->mvdev.max_vqs; i++) { 26741a86b377SEli Cohen mvq = &ndev->vqs[i]; 26751a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 26761a86b377SEli Cohen mvq->index = i; 26771a86b377SEli Cohen mvq->ndev = ndev; 26781a86b377SEli Cohen } 26791a86b377SEli Cohen } 26801a86b377SEli Cohen 268158926c8aSEli Cohen struct mlx5_vdpa_mgmtdev { 268258926c8aSEli Cohen struct vdpa_mgmt_dev mgtdev; 268358926c8aSEli Cohen struct mlx5_adev *madev; 268458926c8aSEli Cohen struct mlx5_vdpa_net *ndev; 268558926c8aSEli Cohen }; 268658926c8aSEli Cohen 2687edf747afSEli Cohen static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) 2688edf747afSEli Cohen { 2689edf747afSEli Cohen u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; 2690edf747afSEli Cohen u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; 2691edf747afSEli Cohen int err; 2692edf747afSEli Cohen 2693edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE); 2694edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, op_mod, opmod); 2695edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, vport_number, vport); 2696edf747afSEli Cohen if (vport) 2697edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, other_vport, 1); 2698edf747afSEli Cohen 2699edf747afSEli Cohen err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out); 2700edf747afSEli Cohen if (err) 2701edf747afSEli Cohen return 0; 2702edf747afSEli Cohen 2703edf747afSEli Cohen return MLX5_GET(query_vport_state_out, out, state); 2704edf747afSEli Cohen } 2705edf747afSEli Cohen 2706edf747afSEli Cohen static bool get_link_state(struct mlx5_vdpa_dev *mvdev) 2707edf747afSEli Cohen { 2708edf747afSEli Cohen if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) == 2709edf747afSEli Cohen VPORT_STATE_UP) 2710edf747afSEli Cohen return true; 2711edf747afSEli Cohen 2712edf747afSEli Cohen return false; 2713edf747afSEli Cohen } 2714edf747afSEli Cohen 2715edf747afSEli Cohen static void update_carrier(struct work_struct *work) 2716edf747afSEli Cohen { 2717edf747afSEli Cohen struct mlx5_vdpa_wq_ent *wqent; 2718edf747afSEli Cohen struct mlx5_vdpa_dev *mvdev; 2719edf747afSEli Cohen struct mlx5_vdpa_net *ndev; 2720edf747afSEli Cohen 2721edf747afSEli Cohen wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 2722edf747afSEli Cohen mvdev = wqent->mvdev; 2723edf747afSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 2724edf747afSEli Cohen if (get_link_state(mvdev)) 2725edf747afSEli Cohen ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 2726edf747afSEli Cohen else 2727edf747afSEli Cohen ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 2728edf747afSEli Cohen 2729edf747afSEli Cohen if (ndev->config_cb.callback) 2730edf747afSEli Cohen ndev->config_cb.callback(ndev->config_cb.private); 2731edf747afSEli Cohen 2732edf747afSEli Cohen kfree(wqent); 2733edf747afSEli Cohen } 2734edf747afSEli Cohen 2735edf747afSEli Cohen static int event_handler(struct notifier_block *nb, unsigned long event, void *param) 2736edf747afSEli Cohen { 2737edf747afSEli Cohen struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb); 2738edf747afSEli Cohen struct mlx5_eqe *eqe = param; 2739edf747afSEli Cohen int ret = NOTIFY_DONE; 2740edf747afSEli Cohen struct mlx5_vdpa_wq_ent *wqent; 2741edf747afSEli Cohen 2742edf747afSEli Cohen if (event == MLX5_EVENT_TYPE_PORT_CHANGE) { 2743edf747afSEli Cohen switch (eqe->sub_type) { 2744edf747afSEli Cohen case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 2745edf747afSEli Cohen case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 2746edf747afSEli Cohen wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); 2747edf747afSEli Cohen if (!wqent) 2748edf747afSEli Cohen return NOTIFY_DONE; 2749edf747afSEli Cohen 2750edf747afSEli Cohen wqent->mvdev = &ndev->mvdev; 2751edf747afSEli Cohen INIT_WORK(&wqent->work, update_carrier); 2752edf747afSEli Cohen queue_work(ndev->mvdev.wq, &wqent->work); 2753edf747afSEli Cohen ret = NOTIFY_OK; 2754edf747afSEli Cohen break; 2755edf747afSEli Cohen default: 2756edf747afSEli Cohen return NOTIFY_DONE; 2757edf747afSEli Cohen } 2758edf747afSEli Cohen return ret; 2759edf747afSEli Cohen } 2760edf747afSEli Cohen return ret; 2761edf747afSEli Cohen } 2762edf747afSEli Cohen 27631e00e821SEli Cohen static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu) 27641e00e821SEli Cohen { 27651e00e821SEli Cohen int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); 27661e00e821SEli Cohen void *in; 27671e00e821SEli Cohen int err; 27681e00e821SEli Cohen 27691e00e821SEli Cohen in = kvzalloc(inlen, GFP_KERNEL); 27701e00e821SEli Cohen if (!in) 27711e00e821SEli Cohen return -ENOMEM; 27721e00e821SEli Cohen 27731e00e821SEli Cohen MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1); 27741e00e821SEli Cohen MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu, 27751e00e821SEli Cohen mtu + MLX5V_ETH_HARD_MTU); 27761e00e821SEli Cohen MLX5_SET(modify_nic_vport_context_in, in, opcode, 27771e00e821SEli Cohen MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); 27781e00e821SEli Cohen 27791e00e821SEli Cohen err = mlx5_cmd_exec_in(mdev, modify_nic_vport_context, in); 27801e00e821SEli Cohen 27811e00e821SEli Cohen kvfree(in); 27821e00e821SEli Cohen return err; 27831e00e821SEli Cohen } 27841e00e821SEli Cohen 2785d8ca2fa5SParav Pandit static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, 2786d8ca2fa5SParav Pandit const struct vdpa_dev_set_config *add_config) 27871a86b377SEli Cohen { 278858926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 27891a86b377SEli Cohen struct virtio_net_config *config; 27907c9f131fSEli Cohen struct mlx5_core_dev *pfmdev; 27911a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev; 27921a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 279358926c8aSEli Cohen struct mlx5_core_dev *mdev; 27941a86b377SEli Cohen u32 max_vqs; 2795246fd1caSEli Cohen u16 mtu; 27961a86b377SEli Cohen int err; 27971a86b377SEli Cohen 279858926c8aSEli Cohen if (mgtdev->ndev) 279958926c8aSEli Cohen return -ENOSPC; 280058926c8aSEli Cohen 280158926c8aSEli Cohen mdev = mgtdev->madev->mdev; 2802879753c8SEli Cohen if (!(MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_queue_type) & 2803879753c8SEli Cohen MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)) { 2804879753c8SEli Cohen dev_warn(mdev->device, "missing support for split virtqueues\n"); 2805879753c8SEli Cohen return -EOPNOTSUPP; 2806879753c8SEli Cohen } 2807879753c8SEli Cohen 2808acde3929SEli Cohen max_vqs = min_t(int, MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues), 2809acde3929SEli Cohen 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size)); 281075560522SEli Cohen if (max_vqs < 2) { 281175560522SEli Cohen dev_warn(mdev->device, 281275560522SEli Cohen "%d virtqueues are supported. At least 2 are required\n", 281375560522SEli Cohen max_vqs); 281475560522SEli Cohen return -EAGAIN; 281575560522SEli Cohen } 281675560522SEli Cohen 281775560522SEli Cohen if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP)) { 281875560522SEli Cohen if (add_config->net.max_vq_pairs > max_vqs / 2) 281975560522SEli Cohen return -EINVAL; 282075560522SEli Cohen max_vqs = min_t(u32, max_vqs, 2 * add_config->net.max_vq_pairs); 282175560522SEli Cohen } else { 282275560522SEli Cohen max_vqs = 2; 282375560522SEli Cohen } 28241a86b377SEli Cohen 28251a86b377SEli Cohen ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, 2826*d4821902SGautam Dawar 1, name, false); 28271a86b377SEli Cohen if (IS_ERR(ndev)) 282874c9729dSLeon Romanovsky return PTR_ERR(ndev); 28291a86b377SEli Cohen 283079de65edSEli Cohen ndev->mvdev.mlx_features = mgtdev->mgtdev.supported_features; 28311a86b377SEli Cohen ndev->mvdev.max_vqs = max_vqs; 28321a86b377SEli Cohen mvdev = &ndev->mvdev; 28331a86b377SEli Cohen mvdev->mdev = mdev; 283475560522SEli Cohen 283575560522SEli Cohen ndev->vqs = kcalloc(max_vqs, sizeof(*ndev->vqs), GFP_KERNEL); 283675560522SEli Cohen ndev->event_cbs = kcalloc(max_vqs + 1, sizeof(*ndev->event_cbs), GFP_KERNEL); 283775560522SEli Cohen if (!ndev->vqs || !ndev->event_cbs) { 283875560522SEli Cohen err = -ENOMEM; 283975560522SEli Cohen goto err_alloc; 284075560522SEli Cohen } 284175560522SEli Cohen 28421a86b377SEli Cohen init_mvqs(ndev); 2843759ae7f9SEli Cohen init_rwsem(&ndev->reslock); 28441a86b377SEli Cohen config = &ndev->config; 28451e00e821SEli Cohen 28461e00e821SEli Cohen if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU)) { 28471e00e821SEli Cohen err = config_func_mtu(mdev, add_config->net.mtu); 28481e00e821SEli Cohen if (err) 2849759ae7f9SEli Cohen goto err_alloc; 28501e00e821SEli Cohen } 28511e00e821SEli Cohen 2852246fd1caSEli Cohen err = query_mtu(mdev, &mtu); 28531a86b377SEli Cohen if (err) 2854759ae7f9SEli Cohen goto err_alloc; 28551a86b377SEli Cohen 2856246fd1caSEli Cohen ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, mtu); 28571a86b377SEli Cohen 2858edf747afSEli Cohen if (get_link_state(mvdev)) 2859edf747afSEli Cohen ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 2860edf747afSEli Cohen else 2861edf747afSEli Cohen ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 2862edf747afSEli Cohen 2863a007d940SEli Cohen if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) { 2864a007d940SEli Cohen memcpy(ndev->config.mac, add_config->net.mac, ETH_ALEN); 2865a007d940SEli Cohen } else { 28661a86b377SEli Cohen err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac); 28671a86b377SEli Cohen if (err) 2868759ae7f9SEli Cohen goto err_alloc; 2869a007d940SEli Cohen } 28701a86b377SEli Cohen 28717c9f131fSEli Cohen if (!is_zero_ether_addr(config->mac)) { 28727c9f131fSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev)); 28737c9f131fSEli Cohen err = mlx5_mpfs_add_mac(pfmdev, config->mac); 28747c9f131fSEli Cohen if (err) 2875759ae7f9SEli Cohen goto err_alloc; 28765262912eSEli Cohen 28775262912eSEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MAC); 28787c9f131fSEli Cohen } 28797c9f131fSEli Cohen 2880acde3929SEli Cohen config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, max_vqs / 2); 28817d23dcdfSEli Cohen mvdev->vdev.dma_dev = &mdev->pdev->dev; 28821a86b377SEli Cohen err = mlx5_vdpa_alloc_resources(&ndev->mvdev); 28831a86b377SEli Cohen if (err) 28847c9f131fSEli Cohen goto err_mpfs; 28851a86b377SEli Cohen 28866f5312f8SEli Cohen if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 28876f5312f8SEli Cohen err = mlx5_vdpa_create_mr(mvdev, NULL); 28881a86b377SEli Cohen if (err) 28891a86b377SEli Cohen goto err_res; 28906f5312f8SEli Cohen } 28916f5312f8SEli Cohen 28926f5312f8SEli Cohen err = alloc_resources(ndev); 28936f5312f8SEli Cohen if (err) 28946f5312f8SEli Cohen goto err_mr; 28951a86b377SEli Cohen 289655ebf0d6SJason Wang ndev->cvq_ent.mvdev = mvdev; 289755ebf0d6SJason Wang INIT_WORK(&ndev->cvq_ent.work, mlx5_cvq_kick_handler); 2898218bdd20SEli Cohen mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_wq"); 28995262912eSEli Cohen if (!mvdev->wq) { 29005262912eSEli Cohen err = -ENOMEM; 29015262912eSEli Cohen goto err_res2; 29025262912eSEli Cohen } 29035262912eSEli Cohen 2904edf747afSEli Cohen ndev->nb.notifier_call = event_handler; 2905edf747afSEli Cohen mlx5_notifier_register(mdev, &ndev->nb); 290658926c8aSEli Cohen mvdev->vdev.mdev = &mgtdev->mgtdev; 2907acde3929SEli Cohen err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1); 29081a86b377SEli Cohen if (err) 29091a86b377SEli Cohen goto err_reg; 29101a86b377SEli Cohen 291158926c8aSEli Cohen mgtdev->ndev = ndev; 291274c9729dSLeon Romanovsky return 0; 29131a86b377SEli Cohen 29141a86b377SEli Cohen err_reg: 29155262912eSEli Cohen destroy_workqueue(mvdev->wq); 29165262912eSEli Cohen err_res2: 29171a86b377SEli Cohen free_resources(ndev); 29186f5312f8SEli Cohen err_mr: 29196f5312f8SEli Cohen mlx5_vdpa_destroy_mr(mvdev); 29201a86b377SEli Cohen err_res: 29211a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 29227c9f131fSEli Cohen err_mpfs: 29237c9f131fSEli Cohen if (!is_zero_ether_addr(config->mac)) 29247c9f131fSEli Cohen mlx5_mpfs_del_mac(pfmdev, config->mac); 292575560522SEli Cohen err_alloc: 29261a86b377SEli Cohen put_device(&mvdev->vdev.dev); 292774c9729dSLeon Romanovsky return err; 29281a86b377SEli Cohen } 29291a86b377SEli Cohen 293058926c8aSEli Cohen static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev) 293158926c8aSEli Cohen { 293258926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 29335262912eSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(dev); 2934edf747afSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2935ad6dc1daSEli Cohen struct workqueue_struct *wq; 293658926c8aSEli Cohen 2937edf747afSEli Cohen mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 2938ad6dc1daSEli Cohen wq = mvdev->wq; 2939ad6dc1daSEli Cohen mvdev->wq = NULL; 2940ad6dc1daSEli Cohen destroy_workqueue(wq); 294158926c8aSEli Cohen _vdpa_unregister_device(dev); 294258926c8aSEli Cohen mgtdev->ndev = NULL; 294358926c8aSEli Cohen } 294458926c8aSEli Cohen 294558926c8aSEli Cohen static const struct vdpa_mgmtdev_ops mdev_ops = { 294658926c8aSEli Cohen .dev_add = mlx5_vdpa_dev_add, 294758926c8aSEli Cohen .dev_del = mlx5_vdpa_dev_del, 294858926c8aSEli Cohen }; 294958926c8aSEli Cohen 295058926c8aSEli Cohen static struct virtio_device_id id_table[] = { 295158926c8aSEli Cohen { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 295258926c8aSEli Cohen { 0 }, 295358926c8aSEli Cohen }; 295458926c8aSEli Cohen 295558926c8aSEli Cohen static int mlx5v_probe(struct auxiliary_device *adev, 295658926c8aSEli Cohen const struct auxiliary_device_id *id) 295758926c8aSEli Cohen 295858926c8aSEli Cohen { 295958926c8aSEli Cohen struct mlx5_adev *madev = container_of(adev, struct mlx5_adev, adev); 296058926c8aSEli Cohen struct mlx5_core_dev *mdev = madev->mdev; 296158926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev; 296258926c8aSEli Cohen int err; 296358926c8aSEli Cohen 296458926c8aSEli Cohen mgtdev = kzalloc(sizeof(*mgtdev), GFP_KERNEL); 296558926c8aSEli Cohen if (!mgtdev) 296658926c8aSEli Cohen return -ENOMEM; 296758926c8aSEli Cohen 296858926c8aSEli Cohen mgtdev->mgtdev.ops = &mdev_ops; 296958926c8aSEli Cohen mgtdev->mgtdev.device = mdev->device; 297058926c8aSEli Cohen mgtdev->mgtdev.id_table = id_table; 297175560522SEli Cohen mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | 29721e00e821SEli Cohen BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP) | 29731e00e821SEli Cohen BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU); 297479de65edSEli Cohen mgtdev->mgtdev.max_supported_vqs = 297579de65edSEli Cohen MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1; 297679de65edSEli Cohen mgtdev->mgtdev.supported_features = get_supported_features(mdev); 297758926c8aSEli Cohen mgtdev->madev = madev; 297858926c8aSEli Cohen 297958926c8aSEli Cohen err = vdpa_mgmtdev_register(&mgtdev->mgtdev); 298058926c8aSEli Cohen if (err) 298158926c8aSEli Cohen goto reg_err; 298258926c8aSEli Cohen 298345e3a279SDavid E. Box auxiliary_set_drvdata(adev, mgtdev); 298458926c8aSEli Cohen 298558926c8aSEli Cohen return 0; 298658926c8aSEli Cohen 298758926c8aSEli Cohen reg_err: 298858926c8aSEli Cohen kfree(mgtdev); 298958926c8aSEli Cohen return err; 299058926c8aSEli Cohen } 299158926c8aSEli Cohen 299274c9729dSLeon Romanovsky static void mlx5v_remove(struct auxiliary_device *adev) 29931a86b377SEli Cohen { 299458926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev; 299574c9729dSLeon Romanovsky 299645e3a279SDavid E. Box mgtdev = auxiliary_get_drvdata(adev); 299758926c8aSEli Cohen vdpa_mgmtdev_unregister(&mgtdev->mgtdev); 299858926c8aSEli Cohen kfree(mgtdev); 29991a86b377SEli Cohen } 300074c9729dSLeon Romanovsky 300174c9729dSLeon Romanovsky static const struct auxiliary_device_id mlx5v_id_table[] = { 300274c9729dSLeon Romanovsky { .name = MLX5_ADEV_NAME ".vnet", }, 300374c9729dSLeon Romanovsky {}, 300474c9729dSLeon Romanovsky }; 300574c9729dSLeon Romanovsky 300674c9729dSLeon Romanovsky MODULE_DEVICE_TABLE(auxiliary, mlx5v_id_table); 300774c9729dSLeon Romanovsky 300874c9729dSLeon Romanovsky static struct auxiliary_driver mlx5v_driver = { 300974c9729dSLeon Romanovsky .name = "vnet", 301074c9729dSLeon Romanovsky .probe = mlx5v_probe, 301174c9729dSLeon Romanovsky .remove = mlx5v_remove, 301274c9729dSLeon Romanovsky .id_table = mlx5v_id_table, 301374c9729dSLeon Romanovsky }; 301474c9729dSLeon Romanovsky 301574c9729dSLeon Romanovsky module_auxiliary_driver(mlx5v_driver); 3016