11a86b377SEli Cohen // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 21a86b377SEli Cohen /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 31a86b377SEli Cohen 474c9729dSLeon Romanovsky #include <linux/module.h> 51a86b377SEli Cohen #include <linux/vdpa.h> 674c9729dSLeon Romanovsky #include <linux/vringh.h> 774c9729dSLeon Romanovsky #include <uapi/linux/virtio_net.h> 81a86b377SEli Cohen #include <uapi/linux/virtio_ids.h> 9a007d940SEli Cohen #include <uapi/linux/vdpa.h> 101a86b377SEli Cohen #include <linux/virtio_config.h> 1174c9729dSLeon Romanovsky #include <linux/auxiliary_bus.h> 1274c9729dSLeon Romanovsky #include <linux/mlx5/cq.h> 131a86b377SEli Cohen #include <linux/mlx5/qp.h> 141a86b377SEli Cohen #include <linux/mlx5/device.h> 1574c9729dSLeon Romanovsky #include <linux/mlx5/driver.h> 161a86b377SEli Cohen #include <linux/mlx5/vport.h> 171a86b377SEli Cohen #include <linux/mlx5/fs.h> 180aae392bSLeon Romanovsky #include <linux/mlx5/mlx5_ifc_vdpa.h> 197c9f131fSEli Cohen #include <linux/mlx5/mpfs.h> 201a86b377SEli Cohen #include "mlx5_vdpa.h" 211a86b377SEli Cohen 2274c9729dSLeon Romanovsky MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 2374c9729dSLeon Romanovsky MODULE_DESCRIPTION("Mellanox VDPA driver"); 2474c9729dSLeon Romanovsky MODULE_LICENSE("Dual BSD/GPL"); 2574c9729dSLeon Romanovsky 2674c9729dSLeon Romanovsky #define to_mlx5_vdpa_ndev(__mvdev) \ 2774c9729dSLeon Romanovsky container_of(__mvdev, struct mlx5_vdpa_net, mvdev) 281a86b377SEli Cohen #define to_mvdev(__vdev) container_of((__vdev), struct mlx5_vdpa_dev, vdev) 291a86b377SEli Cohen 301a86b377SEli Cohen #define VALID_FEATURES_MASK \ 31cbb52359SNathan Chancellor (BIT_ULL(VIRTIO_NET_F_CSUM) | BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | \ 32cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_MAC) | \ 33cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | \ 34cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | \ 35cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | BIT_ULL(VIRTIO_NET_F_HOST_ECN) | BIT_ULL(VIRTIO_NET_F_HOST_UFO) | \ 36cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | BIT_ULL(VIRTIO_NET_F_STATUS) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | \ 37cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_RX) | BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | \ 38cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE) | \ 39cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | \ 40cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_RSS) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY) | \ 41cbb52359SNathan Chancellor BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX) | BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY) | \ 42cbb52359SNathan Chancellor BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | \ 43cbb52359SNathan Chancellor BIT_ULL(VIRTIO_F_RING_PACKED) | BIT_ULL(VIRTIO_F_ORDER_PLATFORM) | BIT_ULL(VIRTIO_F_SR_IOV)) 441a86b377SEli Cohen 451a86b377SEli Cohen #define VALID_STATUS_MASK \ 461a86b377SEli Cohen (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \ 471a86b377SEli Cohen VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED) 481a86b377SEli Cohen 49e4fc6650SEli Cohen #define MLX5_FEATURE(_mvdev, _feature) (!!((_mvdev)->actual_features & BIT_ULL(_feature))) 50e4fc6650SEli Cohen 511a86b377SEli Cohen struct mlx5_vdpa_net_resources { 521a86b377SEli Cohen u32 tisn; 531a86b377SEli Cohen u32 tdn; 541a86b377SEli Cohen u32 tirn; 551a86b377SEli Cohen u32 rqtn; 561a86b377SEli Cohen bool valid; 571a86b377SEli Cohen }; 581a86b377SEli Cohen 591a86b377SEli Cohen struct mlx5_vdpa_cq_buf { 601a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 611a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 621a86b377SEli Cohen int cqe_size; 631a86b377SEli Cohen int nent; 641a86b377SEli Cohen }; 651a86b377SEli Cohen 661a86b377SEli Cohen struct mlx5_vdpa_cq { 671a86b377SEli Cohen struct mlx5_core_cq mcq; 681a86b377SEli Cohen struct mlx5_vdpa_cq_buf buf; 691a86b377SEli Cohen struct mlx5_db db; 701a86b377SEli Cohen int cqe; 711a86b377SEli Cohen }; 721a86b377SEli Cohen 731a86b377SEli Cohen struct mlx5_vdpa_umem { 741a86b377SEli Cohen struct mlx5_frag_buf_ctrl fbc; 751a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 761a86b377SEli Cohen int size; 771a86b377SEli Cohen u32 id; 781a86b377SEli Cohen }; 791a86b377SEli Cohen 801a86b377SEli Cohen struct mlx5_vdpa_qp { 811a86b377SEli Cohen struct mlx5_core_qp mqp; 821a86b377SEli Cohen struct mlx5_frag_buf frag_buf; 831a86b377SEli Cohen struct mlx5_db db; 841a86b377SEli Cohen u16 head; 851a86b377SEli Cohen bool fw; 861a86b377SEli Cohen }; 871a86b377SEli Cohen 881a86b377SEli Cohen struct mlx5_vq_restore_info { 891a86b377SEli Cohen u32 num_ent; 901a86b377SEli Cohen u64 desc_addr; 911a86b377SEli Cohen u64 device_addr; 921a86b377SEli Cohen u64 driver_addr; 931a86b377SEli Cohen u16 avail_index; 94b35ccebeSEli Cohen u16 used_index; 951a86b377SEli Cohen bool ready; 961a86b377SEli Cohen bool restore; 971a86b377SEli Cohen }; 981a86b377SEli Cohen 991a86b377SEli Cohen struct mlx5_vdpa_virtqueue { 1001a86b377SEli Cohen bool ready; 1011a86b377SEli Cohen u64 desc_addr; 1021a86b377SEli Cohen u64 device_addr; 1031a86b377SEli Cohen u64 driver_addr; 1041a86b377SEli Cohen u32 num_ent; 1051a86b377SEli Cohen 1061a86b377SEli Cohen /* Resources for implementing the notification channel from the device 1071a86b377SEli Cohen * to the driver. fwqp is the firmware end of an RC connection; the 1081a86b377SEli Cohen * other end is vqqp used by the driver. cq is is where completions are 1091a86b377SEli Cohen * reported. 1101a86b377SEli Cohen */ 1111a86b377SEli Cohen struct mlx5_vdpa_cq cq; 1121a86b377SEli Cohen struct mlx5_vdpa_qp fwqp; 1131a86b377SEli Cohen struct mlx5_vdpa_qp vqqp; 1141a86b377SEli Cohen 1151a86b377SEli Cohen /* umem resources are required for the virtqueue operation. They're use 1161a86b377SEli Cohen * is internal and they must be provided by the driver. 1171a86b377SEli Cohen */ 1181a86b377SEli Cohen struct mlx5_vdpa_umem umem1; 1191a86b377SEli Cohen struct mlx5_vdpa_umem umem2; 1201a86b377SEli Cohen struct mlx5_vdpa_umem umem3; 1211a86b377SEli Cohen 1221a86b377SEli Cohen bool initialized; 1231a86b377SEli Cohen int index; 1241a86b377SEli Cohen u32 virtq_id; 1251a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 1261a86b377SEli Cohen u16 avail_idx; 127b35ccebeSEli Cohen u16 used_idx; 1281a86b377SEli Cohen int fw_state; 1291a86b377SEli Cohen 1301a86b377SEli Cohen /* keep last in the struct */ 1311a86b377SEli Cohen struct mlx5_vq_restore_info ri; 1321a86b377SEli Cohen }; 1331a86b377SEli Cohen 1341a86b377SEli Cohen /* We will remove this limitation once mlx5_vdpa_alloc_resources() 1351a86b377SEli Cohen * provides for driver space allocation 1361a86b377SEli Cohen */ 1371a86b377SEli Cohen #define MLX5_MAX_SUPPORTED_VQS 16 1381a86b377SEli Cohen 139e4fc6650SEli Cohen static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx) 140e4fc6650SEli Cohen { 141e4fc6650SEli Cohen if (unlikely(idx > mvdev->max_idx)) 142e4fc6650SEli Cohen return false; 143e4fc6650SEli Cohen 144e4fc6650SEli Cohen return true; 145e4fc6650SEli Cohen } 146e4fc6650SEli Cohen 1471a86b377SEli Cohen struct mlx5_vdpa_net { 1481a86b377SEli Cohen struct mlx5_vdpa_dev mvdev; 1491a86b377SEli Cohen struct mlx5_vdpa_net_resources res; 1501a86b377SEli Cohen struct virtio_net_config config; 1511a86b377SEli Cohen struct mlx5_vdpa_virtqueue vqs[MLX5_MAX_SUPPORTED_VQS]; 152db296d25SEli Cohen struct vdpa_callback event_cbs[MLX5_MAX_SUPPORTED_VQS + 1]; 1531a86b377SEli Cohen 1541a86b377SEli Cohen /* Serialize vq resources creation and destruction. This is required 1551a86b377SEli Cohen * since memory map might change and we need to destroy and create 1561a86b377SEli Cohen * resources while driver in operational. 1571a86b377SEli Cohen */ 1581a86b377SEli Cohen struct mutex reslock; 1591a86b377SEli Cohen struct mlx5_flow_table *rxft; 1601a86b377SEli Cohen struct mlx5_fc *rx_counter; 161540061acSEli Cohen struct mlx5_flow_handle *rx_rule_ucast; 162540061acSEli Cohen struct mlx5_flow_handle *rx_rule_mcast; 1631a86b377SEli Cohen bool setup; 1645262912eSEli Cohen u32 cur_num_vqs; 165edf747afSEli Cohen struct notifier_block nb; 166edf747afSEli Cohen struct vdpa_callback config_cb; 1671a86b377SEli Cohen }; 1681a86b377SEli Cohen 1691a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev); 1701a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev); 171ae0428deSEli Cohen static int setup_driver(struct mlx5_vdpa_dev *mvdev); 1721a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev); 1731a86b377SEli Cohen 1741a86b377SEli Cohen static bool mlx5_vdpa_debug; 1751a86b377SEli Cohen 1765262912eSEli Cohen #define MLX5_CVQ_MAX_ENT 16 1775262912eSEli Cohen 1781a86b377SEli Cohen #define MLX5_LOG_VIO_FLAG(_feature) \ 1791a86b377SEli Cohen do { \ 180cbb52359SNathan Chancellor if (features & BIT_ULL(_feature)) \ 1811a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_feature); \ 1821a86b377SEli Cohen } while (0) 1831a86b377SEli Cohen 1841a86b377SEli Cohen #define MLX5_LOG_VIO_STAT(_status) \ 1851a86b377SEli Cohen do { \ 1861a86b377SEli Cohen if (status & (_status)) \ 1871a86b377SEli Cohen mlx5_vdpa_info(mvdev, "%s\n", #_status); \ 1881a86b377SEli Cohen } while (0) 1891a86b377SEli Cohen 19052893733SEli Cohen /* TODO: cross-endian support */ 19152893733SEli Cohen static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev) 19252893733SEli Cohen { 19352893733SEli Cohen return virtio_legacy_is_little_endian() || 19452893733SEli Cohen (mvdev->actual_features & BIT_ULL(VIRTIO_F_VERSION_1)); 19552893733SEli Cohen } 19652893733SEli Cohen 19752893733SEli Cohen static u16 mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev *mvdev, __virtio16 val) 19852893733SEli Cohen { 19952893733SEli Cohen return __virtio16_to_cpu(mlx5_vdpa_is_little_endian(mvdev), val); 20052893733SEli Cohen } 20152893733SEli Cohen 20252893733SEli Cohen static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val) 20352893733SEli Cohen { 20452893733SEli Cohen return __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), val); 20552893733SEli Cohen } 20652893733SEli Cohen 20774c9729dSLeon Romanovsky static inline u32 mlx5_vdpa_max_qps(int max_vqs) 20874c9729dSLeon Romanovsky { 20974c9729dSLeon Romanovsky return max_vqs / 2; 21074c9729dSLeon Romanovsky } 21174c9729dSLeon Romanovsky 2125262912eSEli Cohen static u16 ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev) 2135262912eSEli Cohen { 21452893733SEli Cohen if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) 21552893733SEli Cohen return 2; 21652893733SEli Cohen 2175262912eSEli Cohen return 2 * mlx5_vdpa_max_qps(mvdev->max_vqs); 2185262912eSEli Cohen } 2195262912eSEli Cohen 2205262912eSEli Cohen static bool is_ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev, u16 idx) 2215262912eSEli Cohen { 2225262912eSEli Cohen return idx == ctrl_vq_idx(mvdev); 2235262912eSEli Cohen } 2245262912eSEli Cohen 2251a86b377SEli Cohen static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set) 2261a86b377SEli Cohen { 2271a86b377SEli Cohen if (status & ~VALID_STATUS_MASK) 2281a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n", 2291a86b377SEli Cohen status & ~VALID_STATUS_MASK); 2301a86b377SEli Cohen 2311a86b377SEli Cohen if (!mlx5_vdpa_debug) 2321a86b377SEli Cohen return; 2331a86b377SEli Cohen 2341a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get"); 2351a86b377SEli Cohen if (set && !status) { 2361a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver resets the device\n"); 2371a86b377SEli Cohen return; 2381a86b377SEli Cohen } 2391a86b377SEli Cohen 2401a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE); 2411a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER); 2421a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK); 2431a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK); 2441a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET); 2451a86b377SEli Cohen MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED); 2461a86b377SEli Cohen } 2471a86b377SEli Cohen 2481a86b377SEli Cohen static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set) 2491a86b377SEli Cohen { 2501a86b377SEli Cohen if (features & ~VALID_FEATURES_MASK) 2511a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n", 2521a86b377SEli Cohen features & ~VALID_FEATURES_MASK); 2531a86b377SEli Cohen 2541a86b377SEli Cohen if (!mlx5_vdpa_debug) 2551a86b377SEli Cohen return; 2561a86b377SEli Cohen 2571a86b377SEli Cohen mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads"); 2581a86b377SEli Cohen if (!features) 2591a86b377SEli Cohen mlx5_vdpa_info(mvdev, "all feature bits are cleared\n"); 2601a86b377SEli Cohen 2611a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM); 2621a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM); 2631a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 2641a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU); 2651a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC); 2661a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4); 2671a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6); 2681a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN); 2691a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO); 2701a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4); 2711a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6); 2721a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN); 2731a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO); 2741a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF); 2751a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS); 2761a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ); 2771a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX); 2781a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN); 2791a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA); 2801a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE); 2811a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ); 2821a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR); 2831a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT); 2841a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS); 2851a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT); 2861a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY); 2871a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX); 2881a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY); 2891a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT); 2901a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1); 2911a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM); 2921a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED); 2931a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM); 2941a86b377SEli Cohen MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV); 2951a86b377SEli Cohen } 2961a86b377SEli Cohen 2971a86b377SEli Cohen static int create_tis(struct mlx5_vdpa_net *ndev) 2981a86b377SEli Cohen { 2991a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; 3001a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 3011a86b377SEli Cohen void *tisc; 3021a86b377SEli Cohen int err; 3031a86b377SEli Cohen 3041a86b377SEli Cohen tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3051a86b377SEli Cohen MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); 3061a86b377SEli Cohen err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); 3071a86b377SEli Cohen if (err) 3081a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err); 3091a86b377SEli Cohen 3101a86b377SEli Cohen return err; 3111a86b377SEli Cohen } 3121a86b377SEli Cohen 3131a86b377SEli Cohen static void destroy_tis(struct mlx5_vdpa_net *ndev) 3141a86b377SEli Cohen { 3151a86b377SEli Cohen mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); 3161a86b377SEli Cohen } 3171a86b377SEli Cohen 3181a86b377SEli Cohen #define MLX5_VDPA_CQE_SIZE 64 3191a86b377SEli Cohen #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE) 3201a86b377SEli Cohen 3211a86b377SEli Cohen static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) 3221a86b377SEli Cohen { 3231a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &buf->frag_buf; 3241a86b377SEli Cohen u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE; 3251a86b377SEli Cohen u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE; 3261a86b377SEli Cohen int err; 3271a86b377SEli Cohen 3281a86b377SEli Cohen err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, 3291a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3301a86b377SEli Cohen if (err) 3311a86b377SEli Cohen return err; 3321a86b377SEli Cohen 3331a86b377SEli Cohen mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc); 3341a86b377SEli Cohen 3351a86b377SEli Cohen buf->cqe_size = MLX5_VDPA_CQE_SIZE; 3361a86b377SEli Cohen buf->nent = nent; 3371a86b377SEli Cohen 3381a86b377SEli Cohen return 0; 3391a86b377SEli Cohen } 3401a86b377SEli Cohen 3411a86b377SEli Cohen static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) 3421a86b377SEli Cohen { 3431a86b377SEli Cohen struct mlx5_frag_buf *frag_buf = &umem->frag_buf; 3441a86b377SEli Cohen 3451a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, 3461a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 3471a86b377SEli Cohen } 3481a86b377SEli Cohen 3491a86b377SEli Cohen static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) 3501a86b377SEli Cohen { 3511a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); 3521a86b377SEli Cohen } 3531a86b377SEli Cohen 3541a86b377SEli Cohen static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n) 3551a86b377SEli Cohen { 3561a86b377SEli Cohen return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n); 3571a86b377SEli Cohen } 3581a86b377SEli Cohen 3591a86b377SEli Cohen static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf) 3601a86b377SEli Cohen { 3611a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 3621a86b377SEli Cohen void *cqe; 3631a86b377SEli Cohen int i; 3641a86b377SEli Cohen 3651a86b377SEli Cohen for (i = 0; i < buf->nent; i++) { 3661a86b377SEli Cohen cqe = get_cqe(vcq, i); 3671a86b377SEli Cohen cqe64 = cqe; 3681a86b377SEli Cohen cqe64->op_own = MLX5_CQE_INVALID << 4; 3691a86b377SEli Cohen } 3701a86b377SEli Cohen } 3711a86b377SEli Cohen 3721a86b377SEli Cohen static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n) 3731a86b377SEli Cohen { 3741a86b377SEli Cohen struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1)); 3751a86b377SEli Cohen 3761a86b377SEli Cohen if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && 3771a86b377SEli Cohen !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe))) 3781a86b377SEli Cohen return cqe64; 3791a86b377SEli Cohen 3801a86b377SEli Cohen return NULL; 3811a86b377SEli Cohen } 3821a86b377SEli Cohen 3831a86b377SEli Cohen static void rx_post(struct mlx5_vdpa_qp *vqp, int n) 3841a86b377SEli Cohen { 3851a86b377SEli Cohen vqp->head += n; 3861a86b377SEli Cohen vqp->db.db[0] = cpu_to_be32(vqp->head); 3871a86b377SEli Cohen } 3881a86b377SEli Cohen 3891a86b377SEli Cohen static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, 3901a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq, u32 num_ent) 3911a86b377SEli Cohen { 3921a86b377SEli Cohen struct mlx5_vdpa_qp *vqp; 3931a86b377SEli Cohen __be64 *pas; 3941a86b377SEli Cohen void *qpc; 3951a86b377SEli Cohen 3961a86b377SEli Cohen vqp = fw ? &mvq->fwqp : &mvq->vqqp; 3971a86b377SEli Cohen MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); 3981a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 3991a86b377SEli Cohen if (vqp->fw) { 4001a86b377SEli Cohen /* Firmware QP is allocated by the driver for the firmware's 4011a86b377SEli Cohen * use so we can skip part of the params as they will be chosen by firmware 4021a86b377SEli Cohen */ 4031a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4041a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ); 4051a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 4061a86b377SEli Cohen return; 4071a86b377SEli Cohen } 4081a86b377SEli Cohen 4091a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4101a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4111a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4121a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4131a86b377SEli Cohen MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); 4141a86b377SEli Cohen MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 4151a86b377SEli Cohen MLX5_SET(qpc, qpc, no_sq, 1); 4161a86b377SEli Cohen MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn); 4171a86b377SEli Cohen MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent)); 4181a86b377SEli Cohen MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 4191a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas); 4201a86b377SEli Cohen mlx5_fill_page_frag_array(&vqp->frag_buf, pas); 4211a86b377SEli Cohen } 4221a86b377SEli Cohen 4231a86b377SEli Cohen static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) 4241a86b377SEli Cohen { 4251a86b377SEli Cohen return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, 4261a86b377SEli Cohen num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf, 4271a86b377SEli Cohen ndev->mvdev.mdev->priv.numa_node); 4281a86b377SEli Cohen } 4291a86b377SEli Cohen 4301a86b377SEli Cohen static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 4311a86b377SEli Cohen { 4321a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); 4331a86b377SEli Cohen } 4341a86b377SEli Cohen 4351a86b377SEli Cohen static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 4361a86b377SEli Cohen struct mlx5_vdpa_qp *vqp) 4371a86b377SEli Cohen { 4381a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 4391a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_qp_in); 4401a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 4411a86b377SEli Cohen void *qpc; 4421a86b377SEli Cohen void *in; 4431a86b377SEli Cohen int err; 4441a86b377SEli Cohen 4451a86b377SEli Cohen if (!vqp->fw) { 4461a86b377SEli Cohen vqp = &mvq->vqqp; 4471a86b377SEli Cohen err = rq_buf_alloc(ndev, vqp, mvq->num_ent); 4481a86b377SEli Cohen if (err) 4491a86b377SEli Cohen return err; 4501a86b377SEli Cohen 4511a86b377SEli Cohen err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); 4521a86b377SEli Cohen if (err) 4531a86b377SEli Cohen goto err_db; 4541a86b377SEli Cohen inlen += vqp->frag_buf.npages * sizeof(__be64); 4551a86b377SEli Cohen } 4561a86b377SEli Cohen 4571a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 4581a86b377SEli Cohen if (!in) { 4591a86b377SEli Cohen err = -ENOMEM; 4601a86b377SEli Cohen goto err_kzalloc; 4611a86b377SEli Cohen } 4621a86b377SEli Cohen 4631a86b377SEli Cohen qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); 4641a86b377SEli Cohen qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 4651a86b377SEli Cohen MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 4661a86b377SEli Cohen MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 4671a86b377SEli Cohen MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 4681a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 4691a86b377SEli Cohen if (!vqp->fw) 4701a86b377SEli Cohen MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma); 4711a86b377SEli Cohen MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 4721a86b377SEli Cohen err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 4731a86b377SEli Cohen kfree(in); 4741a86b377SEli Cohen if (err) 4751a86b377SEli Cohen goto err_kzalloc; 4761a86b377SEli Cohen 4771a86b377SEli Cohen vqp->mqp.uid = ndev->mvdev.res.uid; 4781a86b377SEli Cohen vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn); 4791a86b377SEli Cohen 4801a86b377SEli Cohen if (!vqp->fw) 4811a86b377SEli Cohen rx_post(vqp, mvq->num_ent); 4821a86b377SEli Cohen 4831a86b377SEli Cohen return 0; 4841a86b377SEli Cohen 4851a86b377SEli Cohen err_kzalloc: 4861a86b377SEli Cohen if (!vqp->fw) 4871a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 4881a86b377SEli Cohen err_db: 4891a86b377SEli Cohen if (!vqp->fw) 4901a86b377SEli Cohen rq_buf_free(ndev, vqp); 4911a86b377SEli Cohen 4921a86b377SEli Cohen return err; 4931a86b377SEli Cohen } 4941a86b377SEli Cohen 4951a86b377SEli Cohen static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 4961a86b377SEli Cohen { 4971a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 4981a86b377SEli Cohen 4991a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 5001a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn); 5011a86b377SEli Cohen MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); 5021a86b377SEli Cohen if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) 5031a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); 5041a86b377SEli Cohen if (!vqp->fw) { 5051a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 5061a86b377SEli Cohen rq_buf_free(ndev, vqp); 5071a86b377SEli Cohen } 5081a86b377SEli Cohen } 5091a86b377SEli Cohen 5101a86b377SEli Cohen static void *next_cqe_sw(struct mlx5_vdpa_cq *cq) 5111a86b377SEli Cohen { 5121a86b377SEli Cohen return get_sw_cqe(cq, cq->mcq.cons_index); 5131a86b377SEli Cohen } 5141a86b377SEli Cohen 5151a86b377SEli Cohen static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq) 5161a86b377SEli Cohen { 5171a86b377SEli Cohen struct mlx5_cqe64 *cqe64; 5181a86b377SEli Cohen 5191a86b377SEli Cohen cqe64 = next_cqe_sw(vcq); 5201a86b377SEli Cohen if (!cqe64) 5211a86b377SEli Cohen return -EAGAIN; 5221a86b377SEli Cohen 5231a86b377SEli Cohen vcq->mcq.cons_index++; 5241a86b377SEli Cohen return 0; 5251a86b377SEli Cohen } 5261a86b377SEli Cohen 5271a86b377SEli Cohen static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num) 5281a86b377SEli Cohen { 529db296d25SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 530db296d25SEli Cohen struct vdpa_callback *event_cb; 531db296d25SEli Cohen 532db296d25SEli Cohen event_cb = &ndev->event_cbs[mvq->index]; 5331a86b377SEli Cohen mlx5_cq_set_ci(&mvq->cq.mcq); 53483ef73b2SEli Cohen 53583ef73b2SEli Cohen /* make sure CQ cosumer update is visible to the hardware before updating 53683ef73b2SEli Cohen * RX doorbell record. 53783ef73b2SEli Cohen */ 53883ef73b2SEli Cohen dma_wmb(); 5391a86b377SEli Cohen rx_post(&mvq->vqqp, num); 540db296d25SEli Cohen if (event_cb->callback) 541db296d25SEli Cohen event_cb->callback(event_cb->private); 5421a86b377SEli Cohen } 5431a86b377SEli Cohen 5441a86b377SEli Cohen static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) 5451a86b377SEli Cohen { 5461a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq); 5471a86b377SEli Cohen struct mlx5_vdpa_net *ndev = mvq->ndev; 5481a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5491a86b377SEli Cohen int num = 0; 5501a86b377SEli Cohen 5511a86b377SEli Cohen while (!mlx5_vdpa_poll_one(&mvq->cq)) { 5521a86b377SEli Cohen num++; 5531a86b377SEli Cohen if (num > mvq->num_ent / 2) { 5541a86b377SEli Cohen /* If completions keep coming while we poll, we want to 5551a86b377SEli Cohen * let the hardware know that we consumed them by 5561a86b377SEli Cohen * updating the doorbell record. We also let vdpa core 5571a86b377SEli Cohen * know about this so it passes it on the virtio driver 5581a86b377SEli Cohen * on the guest. 5591a86b377SEli Cohen */ 5601a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 5611a86b377SEli Cohen num = 0; 5621a86b377SEli Cohen } 5631a86b377SEli Cohen } 5641a86b377SEli Cohen 5651a86b377SEli Cohen if (num) 5661a86b377SEli Cohen mlx5_vdpa_handle_completions(mvq, num); 5671a86b377SEli Cohen 5681a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 5691a86b377SEli Cohen } 5701a86b377SEli Cohen 5711a86b377SEli Cohen static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) 5721a86b377SEli Cohen { 5731a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 5741a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 5751a86b377SEli Cohen void __iomem *uar_page = ndev->mvdev.res.uar->map; 5761a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 5771a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 5781a86b377SEli Cohen __be64 *pas; 5791a86b377SEli Cohen int inlen; 5801a86b377SEli Cohen void *cqc; 5811a86b377SEli Cohen void *in; 5821a86b377SEli Cohen int err; 5831a86b377SEli Cohen int eqn; 5841a86b377SEli Cohen 5851a86b377SEli Cohen err = mlx5_db_alloc(mdev, &vcq->db); 5861a86b377SEli Cohen if (err) 5871a86b377SEli Cohen return err; 5881a86b377SEli Cohen 5891a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 5901a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 5911a86b377SEli Cohen vcq->mcq.cqe_sz = 64; 5921a86b377SEli Cohen 5931a86b377SEli Cohen err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); 5941a86b377SEli Cohen if (err) 5951a86b377SEli Cohen goto err_db; 5961a86b377SEli Cohen 5971a86b377SEli Cohen cq_frag_buf_init(vcq, &vcq->buf); 5981a86b377SEli Cohen 5991a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 6001a86b377SEli Cohen MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages; 6011a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 6021a86b377SEli Cohen if (!in) { 6031a86b377SEli Cohen err = -ENOMEM; 6041a86b377SEli Cohen goto err_vzalloc; 6051a86b377SEli Cohen } 6061a86b377SEli Cohen 6071a86b377SEli Cohen MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); 6081a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 6091a86b377SEli Cohen mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas); 6101a86b377SEli Cohen 6111a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 6121a86b377SEli Cohen MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 6131a86b377SEli Cohen 6141a86b377SEli Cohen /* Use vector 0 by default. Consider adding code to choose least used 6151a86b377SEli Cohen * vector. 6161a86b377SEli Cohen */ 617563476aeSShay Drory err = mlx5_vector2eqn(mdev, 0, &eqn); 6181a86b377SEli Cohen if (err) 6191a86b377SEli Cohen goto err_vec; 6201a86b377SEli Cohen 6211a86b377SEli Cohen cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 6221a86b377SEli Cohen MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent)); 6231a86b377SEli Cohen MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); 624616d5769STal Gilboa MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 6251a86b377SEli Cohen MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma); 6261a86b377SEli Cohen 6271a86b377SEli Cohen err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out)); 6281a86b377SEli Cohen if (err) 6291a86b377SEli Cohen goto err_vec; 6301a86b377SEli Cohen 6311a86b377SEli Cohen vcq->mcq.comp = mlx5_vdpa_cq_comp; 6321a86b377SEli Cohen vcq->cqe = num_ent; 6331a86b377SEli Cohen vcq->mcq.set_ci_db = vcq->db.db; 6341a86b377SEli Cohen vcq->mcq.arm_db = vcq->db.db + 1; 6351a86b377SEli Cohen mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 6361a86b377SEli Cohen kfree(in); 6371a86b377SEli Cohen return 0; 6381a86b377SEli Cohen 6391a86b377SEli Cohen err_vec: 6401a86b377SEli Cohen kfree(in); 6411a86b377SEli Cohen err_vzalloc: 6421a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 6431a86b377SEli Cohen err_db: 6441a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 6451a86b377SEli Cohen return err; 6461a86b377SEli Cohen } 6471a86b377SEli Cohen 6481a86b377SEli Cohen static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) 6491a86b377SEli Cohen { 6501a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 6511a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 6521a86b377SEli Cohen struct mlx5_vdpa_cq *vcq = &mvq->cq; 6531a86b377SEli Cohen 6541a86b377SEli Cohen if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) { 6551a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); 6561a86b377SEli Cohen return; 6571a86b377SEli Cohen } 6581a86b377SEli Cohen cq_frag_buf_free(ndev, &vcq->buf); 6591a86b377SEli Cohen mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 6601a86b377SEli Cohen } 6611a86b377SEli Cohen 66271ab6a7cSEli Cohen static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, 6631a86b377SEli Cohen struct mlx5_vdpa_umem **umemp) 6641a86b377SEli Cohen { 6651a86b377SEli Cohen struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 6661a86b377SEli Cohen int p_a; 6671a86b377SEli Cohen int p_b; 6681a86b377SEli Cohen 6691a86b377SEli Cohen switch (num) { 6701a86b377SEli Cohen case 1: 6711a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a); 6721a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b); 6731a86b377SEli Cohen *umemp = &mvq->umem1; 6741a86b377SEli Cohen break; 6751a86b377SEli Cohen case 2: 6761a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a); 6771a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b); 6781a86b377SEli Cohen *umemp = &mvq->umem2; 6791a86b377SEli Cohen break; 6801a86b377SEli Cohen case 3: 6811a86b377SEli Cohen p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a); 6821a86b377SEli Cohen p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b); 6831a86b377SEli Cohen *umemp = &mvq->umem3; 6841a86b377SEli Cohen break; 6851a86b377SEli Cohen } 68671ab6a7cSEli Cohen (*umemp)->size = p_a * mvq->num_ent + p_b; 6871a86b377SEli Cohen } 6881a86b377SEli Cohen 6891a86b377SEli Cohen static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) 6901a86b377SEli Cohen { 6911a86b377SEli Cohen mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); 6921a86b377SEli Cohen } 6931a86b377SEli Cohen 6941a86b377SEli Cohen static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 6951a86b377SEli Cohen { 6961a86b377SEli Cohen int inlen; 6971a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {}; 6981a86b377SEli Cohen void *um; 6991a86b377SEli Cohen void *in; 7001a86b377SEli Cohen int err; 7011a86b377SEli Cohen __be64 *pas; 7021a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 7031a86b377SEli Cohen 70471ab6a7cSEli Cohen set_umem_size(ndev, mvq, num, &umem); 70571ab6a7cSEli Cohen err = umem_frag_buf_alloc(ndev, umem, umem->size); 7061a86b377SEli Cohen if (err) 7071a86b377SEli Cohen return err; 7081a86b377SEli Cohen 7091a86b377SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages; 7101a86b377SEli Cohen 7111a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 7121a86b377SEli Cohen if (!in) { 7131a86b377SEli Cohen err = -ENOMEM; 7141a86b377SEli Cohen goto err_in; 7151a86b377SEli Cohen } 7161a86b377SEli Cohen 7171a86b377SEli Cohen MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM); 7181a86b377SEli Cohen MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); 7191a86b377SEli Cohen um = MLX5_ADDR_OF(create_umem_in, in, umem); 7201a86b377SEli Cohen MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 7211a86b377SEli Cohen MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages); 7221a86b377SEli Cohen 7231a86b377SEli Cohen pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]); 7241a86b377SEli Cohen mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW); 7251a86b377SEli Cohen 7261a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 7271a86b377SEli Cohen if (err) { 7281a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); 7291a86b377SEli Cohen goto err_cmd; 7301a86b377SEli Cohen } 7311a86b377SEli Cohen 7321a86b377SEli Cohen kfree(in); 7331a86b377SEli Cohen umem->id = MLX5_GET(create_umem_out, out, umem_id); 7341a86b377SEli Cohen 7351a86b377SEli Cohen return 0; 7361a86b377SEli Cohen 7371a86b377SEli Cohen err_cmd: 7381a86b377SEli Cohen kfree(in); 7391a86b377SEli Cohen err_in: 7401a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7411a86b377SEli Cohen return err; 7421a86b377SEli Cohen } 7431a86b377SEli Cohen 7441a86b377SEli Cohen static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 7451a86b377SEli Cohen { 7461a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {}; 7471a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {}; 7481a86b377SEli Cohen struct mlx5_vdpa_umem *umem; 7491a86b377SEli Cohen 7501a86b377SEli Cohen switch (num) { 7511a86b377SEli Cohen case 1: 7521a86b377SEli Cohen umem = &mvq->umem1; 7531a86b377SEli Cohen break; 7541a86b377SEli Cohen case 2: 7551a86b377SEli Cohen umem = &mvq->umem2; 7561a86b377SEli Cohen break; 7571a86b377SEli Cohen case 3: 7581a86b377SEli Cohen umem = &mvq->umem3; 7591a86b377SEli Cohen break; 7601a86b377SEli Cohen } 7611a86b377SEli Cohen 7621a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM); 7631a86b377SEli Cohen MLX5_SET(destroy_umem_in, in, umem_id, umem->id); 7641a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 7651a86b377SEli Cohen return; 7661a86b377SEli Cohen 7671a86b377SEli Cohen umem_frag_buf_free(ndev, umem); 7681a86b377SEli Cohen } 7691a86b377SEli Cohen 7701a86b377SEli Cohen static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7711a86b377SEli Cohen { 7721a86b377SEli Cohen int num; 7731a86b377SEli Cohen int err; 7741a86b377SEli Cohen 7751a86b377SEli Cohen for (num = 1; num <= 3; num++) { 7761a86b377SEli Cohen err = create_umem(ndev, mvq, num); 7771a86b377SEli Cohen if (err) 7781a86b377SEli Cohen goto err_umem; 7791a86b377SEli Cohen } 7801a86b377SEli Cohen return 0; 7811a86b377SEli Cohen 7821a86b377SEli Cohen err_umem: 7831a86b377SEli Cohen for (num--; num > 0; num--) 7841a86b377SEli Cohen umem_destroy(ndev, mvq, num); 7851a86b377SEli Cohen 7861a86b377SEli Cohen return err; 7871a86b377SEli Cohen } 7881a86b377SEli Cohen 7891a86b377SEli Cohen static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 7901a86b377SEli Cohen { 7911a86b377SEli Cohen int num; 7921a86b377SEli Cohen 7931a86b377SEli Cohen for (num = 3; num > 0; num--) 7941a86b377SEli Cohen umem_destroy(ndev, mvq, num); 7951a86b377SEli Cohen } 7961a86b377SEli Cohen 7971a86b377SEli Cohen static int get_queue_type(struct mlx5_vdpa_net *ndev) 7981a86b377SEli Cohen { 7991a86b377SEli Cohen u32 type_mask; 8001a86b377SEli Cohen 8011a86b377SEli Cohen type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); 8021a86b377SEli Cohen 8031a86b377SEli Cohen /* prefer split queue */ 804879753c8SEli Cohen if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT) 8051a86b377SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT; 806879753c8SEli Cohen 807879753c8SEli Cohen WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED)); 808879753c8SEli Cohen 809879753c8SEli Cohen return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED; 8101a86b377SEli Cohen } 8111a86b377SEli Cohen 8121a86b377SEli Cohen static bool vq_is_tx(u16 idx) 8131a86b377SEli Cohen { 8141a86b377SEli Cohen return idx % 2; 8151a86b377SEli Cohen } 8161a86b377SEli Cohen 8171a86b377SEli Cohen static u16 get_features_12_3(u64 features) 8181a86b377SEli Cohen { 819cbb52359SNathan Chancellor return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) | 820cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) | 821cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) | 822cbb52359SNathan Chancellor (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6); 8231a86b377SEli Cohen } 8241a86b377SEli Cohen 8251a86b377SEli Cohen static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 8261a86b377SEli Cohen { 8271a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); 8281a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {}; 8291a86b377SEli Cohen void *obj_context; 8301a86b377SEli Cohen void *cmd_hdr; 8311a86b377SEli Cohen void *vq_ctx; 8321a86b377SEli Cohen void *in; 8331a86b377SEli Cohen int err; 8341a86b377SEli Cohen 8351a86b377SEli Cohen err = umems_create(ndev, mvq); 8361a86b377SEli Cohen if (err) 8371a86b377SEli Cohen return err; 8381a86b377SEli Cohen 8391a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 8401a86b377SEli Cohen if (!in) { 8411a86b377SEli Cohen err = -ENOMEM; 8421a86b377SEli Cohen goto err_alloc; 8431a86b377SEli Cohen } 8441a86b377SEli Cohen 8451a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr); 8461a86b377SEli Cohen 8471a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 8481a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 8491a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 8501a86b377SEli Cohen 8511a86b377SEli Cohen obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context); 8521a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); 853b35ccebeSEli Cohen MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); 8541a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, 8551a86b377SEli Cohen get_features_12_3(ndev->mvdev.actual_features)); 8561a86b377SEli Cohen vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context); 8571a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); 8581a86b377SEli Cohen 8591a86b377SEli Cohen if (vq_is_tx(mvq->index)) 8601a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); 8611a86b377SEli Cohen 8621a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 8631a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); 8641a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 8651a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); 8661a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 8674b454a82SEli Cohen !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); 8681a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); 8691a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); 8701a86b377SEli Cohen MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); 87183fec3f1SAharon Landau MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey); 8721a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); 8731a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); 8741a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id); 875e3011776SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem2.size); 8761a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id); 877e3011776SEli Cohen MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem3.size); 8781a86b377SEli Cohen MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); 8791a86b377SEli Cohen 8801a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 8811a86b377SEli Cohen if (err) 8821a86b377SEli Cohen goto err_cmd; 8831a86b377SEli Cohen 8841a86b377SEli Cohen kfree(in); 8851a86b377SEli Cohen mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 8861a86b377SEli Cohen 8871a86b377SEli Cohen return 0; 8881a86b377SEli Cohen 8891a86b377SEli Cohen err_cmd: 8901a86b377SEli Cohen kfree(in); 8911a86b377SEli Cohen err_alloc: 8921a86b377SEli Cohen umems_destroy(ndev, mvq); 8931a86b377SEli Cohen return err; 8941a86b377SEli Cohen } 8951a86b377SEli Cohen 8961a86b377SEli Cohen static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 8971a86b377SEli Cohen { 8981a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {}; 8991a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {}; 9001a86b377SEli Cohen 9011a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode, 9021a86b377SEli Cohen MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 9031a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id); 9041a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); 9051a86b377SEli Cohen MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type, 9061a86b377SEli Cohen MLX5_OBJ_TYPE_VIRTIO_NET_Q); 9071a86b377SEli Cohen if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { 9081a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); 9091a86b377SEli Cohen return; 9101a86b377SEli Cohen } 9111a86b377SEli Cohen umems_destroy(ndev, mvq); 9121a86b377SEli Cohen } 9131a86b377SEli Cohen 9141a86b377SEli Cohen static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 9151a86b377SEli Cohen { 9161a86b377SEli Cohen return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn; 9171a86b377SEli Cohen } 9181a86b377SEli Cohen 9191a86b377SEli Cohen static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 9201a86b377SEli Cohen { 9211a86b377SEli Cohen return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn; 9221a86b377SEli Cohen } 9231a86b377SEli Cohen 9241a86b377SEli Cohen static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, 9251a86b377SEli Cohen int *outlen, u32 qpn, u32 rqpn) 9261a86b377SEli Cohen { 9271a86b377SEli Cohen void *qpc; 9281a86b377SEli Cohen void *pp; 9291a86b377SEli Cohen 9301a86b377SEli Cohen switch (cmd) { 9311a86b377SEli Cohen case MLX5_CMD_OP_2RST_QP: 9321a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in); 9331a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out); 9341a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9351a86b377SEli Cohen *out = kzalloc(*outlen, GFP_KERNEL); 936f31231bfSColin Ian King if (!*in || !*out) 9371a86b377SEli Cohen goto outerr; 9381a86b377SEli Cohen 9391a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, opcode, cmd); 9401a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); 9411a86b377SEli Cohen MLX5_SET(qp_2rst_in, *in, qpn, qpn); 9421a86b377SEli Cohen break; 9431a86b377SEli Cohen case MLX5_CMD_OP_RST2INIT_QP: 9441a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in); 9451a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out); 9461a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9471a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL); 948f31231bfSColin Ian King if (!*in || !*out) 9491a86b377SEli Cohen goto outerr; 9501a86b377SEli Cohen 9511a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, opcode, cmd); 9521a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); 9531a86b377SEli Cohen MLX5_SET(rst2init_qp_in, *in, qpn, qpn); 9541a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9551a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9561a86b377SEli Cohen MLX5_SET(qpc, qpc, rwe, 1); 9571a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9581a86b377SEli Cohen MLX5_SET(ads, pp, vhca_port_num, 1); 9591a86b377SEli Cohen break; 9601a86b377SEli Cohen case MLX5_CMD_OP_INIT2RTR_QP: 9611a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in); 9621a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out); 9631a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9641a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL); 965f31231bfSColin Ian King if (!*in || !*out) 9661a86b377SEli Cohen goto outerr; 9671a86b377SEli Cohen 9681a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, opcode, cmd); 9691a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); 9701a86b377SEli Cohen MLX5_SET(init2rtr_qp_in, *in, qpn, qpn); 9711a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9721a86b377SEli Cohen MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 9731a86b377SEli Cohen MLX5_SET(qpc, qpc, log_msg_max, 30); 9741a86b377SEli Cohen MLX5_SET(qpc, qpc, remote_qpn, rqpn); 9751a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9761a86b377SEli Cohen MLX5_SET(ads, pp, fl, 1); 9771a86b377SEli Cohen break; 9781a86b377SEli Cohen case MLX5_CMD_OP_RTR2RTS_QP: 9791a86b377SEli Cohen *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in); 9801a86b377SEli Cohen *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out); 9811a86b377SEli Cohen *in = kzalloc(*inlen, GFP_KERNEL); 9821a86b377SEli Cohen *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL); 983f31231bfSColin Ian King if (!*in || !*out) 9841a86b377SEli Cohen goto outerr; 9851a86b377SEli Cohen 9861a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd); 9871a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); 9881a86b377SEli Cohen MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn); 9891a86b377SEli Cohen qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 9901a86b377SEli Cohen pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 9911a86b377SEli Cohen MLX5_SET(ads, pp, ack_timeout, 14); 9921a86b377SEli Cohen MLX5_SET(qpc, qpc, retry_count, 7); 9931a86b377SEli Cohen MLX5_SET(qpc, qpc, rnr_retry, 7); 9941a86b377SEli Cohen break; 9951a86b377SEli Cohen default: 996f31231bfSColin Ian King goto outerr_nullify; 9971a86b377SEli Cohen } 9981a86b377SEli Cohen 9991a86b377SEli Cohen return; 10001a86b377SEli Cohen 10011a86b377SEli Cohen outerr: 10021a86b377SEli Cohen kfree(*in); 10031a86b377SEli Cohen kfree(*out); 1004f31231bfSColin Ian King outerr_nullify: 10051a86b377SEli Cohen *in = NULL; 10061a86b377SEli Cohen *out = NULL; 10071a86b377SEli Cohen } 10081a86b377SEli Cohen 10091a86b377SEli Cohen static void free_inout(void *in, void *out) 10101a86b377SEli Cohen { 10111a86b377SEli Cohen kfree(in); 10121a86b377SEli Cohen kfree(out); 10131a86b377SEli Cohen } 10141a86b377SEli Cohen 10151a86b377SEli Cohen /* Two QPs are used by each virtqueue. One is used by the driver and one by 10161a86b377SEli Cohen * firmware. The fw argument indicates whether the subjected QP is the one used 10171a86b377SEli Cohen * by firmware. 10181a86b377SEli Cohen */ 10191a86b377SEli Cohen static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) 10201a86b377SEli Cohen { 10211a86b377SEli Cohen int outlen; 10221a86b377SEli Cohen int inlen; 10231a86b377SEli Cohen void *out; 10241a86b377SEli Cohen void *in; 10251a86b377SEli Cohen int err; 10261a86b377SEli Cohen 10271a86b377SEli Cohen alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); 10281a86b377SEli Cohen if (!in || !out) 10291a86b377SEli Cohen return -ENOMEM; 10301a86b377SEli Cohen 10311a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); 10321a86b377SEli Cohen free_inout(in, out); 10331a86b377SEli Cohen return err; 10341a86b377SEli Cohen } 10351a86b377SEli Cohen 10361a86b377SEli Cohen static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 10371a86b377SEli Cohen { 10381a86b377SEli Cohen int err; 10391a86b377SEli Cohen 10401a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); 10411a86b377SEli Cohen if (err) 10421a86b377SEli Cohen return err; 10431a86b377SEli Cohen 10441a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); 10451a86b377SEli Cohen if (err) 10461a86b377SEli Cohen return err; 10471a86b377SEli Cohen 10481a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); 10491a86b377SEli Cohen if (err) 10501a86b377SEli Cohen return err; 10511a86b377SEli Cohen 10521a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); 10531a86b377SEli Cohen if (err) 10541a86b377SEli Cohen return err; 10551a86b377SEli Cohen 10561a86b377SEli Cohen err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); 10571a86b377SEli Cohen if (err) 10581a86b377SEli Cohen return err; 10591a86b377SEli Cohen 10601a86b377SEli Cohen err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); 10611a86b377SEli Cohen if (err) 10621a86b377SEli Cohen return err; 10631a86b377SEli Cohen 10641a86b377SEli Cohen return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); 10651a86b377SEli Cohen } 10661a86b377SEli Cohen 10671a86b377SEli Cohen struct mlx5_virtq_attr { 10681a86b377SEli Cohen u8 state; 10691a86b377SEli Cohen u16 available_index; 1070b35ccebeSEli Cohen u16 used_index; 10711a86b377SEli Cohen }; 10721a86b377SEli Cohen 10731a86b377SEli Cohen static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 10741a86b377SEli Cohen struct mlx5_virtq_attr *attr) 10751a86b377SEli Cohen { 10761a86b377SEli Cohen int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out); 10771a86b377SEli Cohen u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {}; 10781a86b377SEli Cohen void *out; 10791a86b377SEli Cohen void *obj_context; 10801a86b377SEli Cohen void *cmd_hdr; 10811a86b377SEli Cohen int err; 10821a86b377SEli Cohen 10831a86b377SEli Cohen out = kzalloc(outlen, GFP_KERNEL); 10841a86b377SEli Cohen if (!out) 10851a86b377SEli Cohen return -ENOMEM; 10861a86b377SEli Cohen 10871a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr); 10881a86b377SEli Cohen 10891a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 10901a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 10911a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 10921a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 10931a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen); 10941a86b377SEli Cohen if (err) 10951a86b377SEli Cohen goto err_cmd; 10961a86b377SEli Cohen 10971a86b377SEli Cohen obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context); 10981a86b377SEli Cohen memset(attr, 0, sizeof(*attr)); 10991a86b377SEli Cohen attr->state = MLX5_GET(virtio_net_q_object, obj_context, state); 11001a86b377SEli Cohen attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index); 1101b35ccebeSEli Cohen attr->used_index = MLX5_GET(virtio_net_q_object, obj_context, hw_used_index); 11021a86b377SEli Cohen kfree(out); 11031a86b377SEli Cohen return 0; 11041a86b377SEli Cohen 11051a86b377SEli Cohen err_cmd: 11061a86b377SEli Cohen kfree(out); 11071a86b377SEli Cohen return err; 11081a86b377SEli Cohen } 11091a86b377SEli Cohen 11101a86b377SEli Cohen static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) 11111a86b377SEli Cohen { 11121a86b377SEli Cohen int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); 11131a86b377SEli Cohen u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {}; 11141a86b377SEli Cohen void *obj_context; 11151a86b377SEli Cohen void *cmd_hdr; 11161a86b377SEli Cohen void *in; 11171a86b377SEli Cohen int err; 11181a86b377SEli Cohen 11191a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 11201a86b377SEli Cohen if (!in) 11211a86b377SEli Cohen return -ENOMEM; 11221a86b377SEli Cohen 11231a86b377SEli Cohen cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr); 11241a86b377SEli Cohen 11251a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); 11261a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 11271a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 11281a86b377SEli Cohen MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 11291a86b377SEli Cohen 11301a86b377SEli Cohen obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context); 11311a86b377SEli Cohen MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select, 11321a86b377SEli Cohen MLX5_VIRTQ_MODIFY_MASK_STATE); 11331a86b377SEli Cohen MLX5_SET(virtio_net_q_object, obj_context, state, state); 11341a86b377SEli Cohen err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 11351a86b377SEli Cohen kfree(in); 11361a86b377SEli Cohen if (!err) 11371a86b377SEli Cohen mvq->fw_state = state; 11381a86b377SEli Cohen 11391a86b377SEli Cohen return err; 11401a86b377SEli Cohen } 11411a86b377SEli Cohen 11421a86b377SEli Cohen static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11431a86b377SEli Cohen { 11441a86b377SEli Cohen u16 idx = mvq->index; 11451a86b377SEli Cohen int err; 11461a86b377SEli Cohen 11471a86b377SEli Cohen if (!mvq->num_ent) 11481a86b377SEli Cohen return 0; 11491a86b377SEli Cohen 115052893733SEli Cohen if (mvq->initialized) 115152893733SEli Cohen return 0; 11521a86b377SEli Cohen 11531a86b377SEli Cohen err = cq_create(ndev, idx, mvq->num_ent); 11541a86b377SEli Cohen if (err) 11551a86b377SEli Cohen return err; 11561a86b377SEli Cohen 11571a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->fwqp); 11581a86b377SEli Cohen if (err) 11591a86b377SEli Cohen goto err_fwqp; 11601a86b377SEli Cohen 11611a86b377SEli Cohen err = qp_create(ndev, mvq, &mvq->vqqp); 11621a86b377SEli Cohen if (err) 11631a86b377SEli Cohen goto err_vqqp; 11641a86b377SEli Cohen 11651a86b377SEli Cohen err = connect_qps(ndev, mvq); 11661a86b377SEli Cohen if (err) 11671a86b377SEli Cohen goto err_connect; 11681a86b377SEli Cohen 11691a86b377SEli Cohen err = create_virtqueue(ndev, mvq); 11701a86b377SEli Cohen if (err) 11711a86b377SEli Cohen goto err_connect; 11721a86b377SEli Cohen 11731a86b377SEli Cohen if (mvq->ready) { 11741a86b377SEli Cohen err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 11751a86b377SEli Cohen if (err) { 11761a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", 11771a86b377SEli Cohen idx, err); 11781a86b377SEli Cohen goto err_connect; 11791a86b377SEli Cohen } 11801a86b377SEli Cohen } 11811a86b377SEli Cohen 11821a86b377SEli Cohen mvq->initialized = true; 11831a86b377SEli Cohen return 0; 11841a86b377SEli Cohen 11851a86b377SEli Cohen err_connect: 11861a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 11871a86b377SEli Cohen err_vqqp: 11881a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 11891a86b377SEli Cohen err_fwqp: 11901a86b377SEli Cohen cq_destroy(ndev, idx); 11911a86b377SEli Cohen return err; 11921a86b377SEli Cohen } 11931a86b377SEli Cohen 11941a86b377SEli Cohen static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 11951a86b377SEli Cohen { 11961a86b377SEli Cohen struct mlx5_virtq_attr attr; 11971a86b377SEli Cohen 11981a86b377SEli Cohen if (!mvq->initialized) 11991a86b377SEli Cohen return; 12001a86b377SEli Cohen 12011a86b377SEli Cohen if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 12021a86b377SEli Cohen return; 12031a86b377SEli Cohen 12041a86b377SEli Cohen if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) 12051a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); 12063176e974SSi-Wei Liu 12073176e974SSi-Wei Liu if (query_virtqueue(ndev, mvq, &attr)) { 12083176e974SSi-Wei Liu mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); 12093176e974SSi-Wei Liu return; 12103176e974SSi-Wei Liu } 12113176e974SSi-Wei Liu mvq->avail_idx = attr.available_index; 1212bc04d93eSEli Cohen mvq->used_idx = attr.used_index; 12131a86b377SEli Cohen } 12141a86b377SEli Cohen 12151a86b377SEli Cohen static void suspend_vqs(struct mlx5_vdpa_net *ndev) 12161a86b377SEli Cohen { 12171a86b377SEli Cohen int i; 12181a86b377SEli Cohen 12191a86b377SEli Cohen for (i = 0; i < MLX5_MAX_SUPPORTED_VQS; i++) 12201a86b377SEli Cohen suspend_vq(ndev, &ndev->vqs[i]); 12211a86b377SEli Cohen } 12221a86b377SEli Cohen 12231a86b377SEli Cohen static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 12241a86b377SEli Cohen { 12251a86b377SEli Cohen if (!mvq->initialized) 12261a86b377SEli Cohen return; 12271a86b377SEli Cohen 12281a86b377SEli Cohen suspend_vq(ndev, mvq); 12291a86b377SEli Cohen destroy_virtqueue(ndev, mvq); 12301a86b377SEli Cohen qp_destroy(ndev, &mvq->vqqp); 12311a86b377SEli Cohen qp_destroy(ndev, &mvq->fwqp); 12321a86b377SEli Cohen cq_destroy(ndev, mvq->index); 12331a86b377SEli Cohen mvq->initialized = false; 12341a86b377SEli Cohen } 12351a86b377SEli Cohen 12361a86b377SEli Cohen static int create_rqt(struct mlx5_vdpa_net *ndev) 12371a86b377SEli Cohen { 12381a86b377SEli Cohen __be32 *list; 123952893733SEli Cohen int max_rqt; 12401a86b377SEli Cohen void *rqtc; 12411a86b377SEli Cohen int inlen; 12421a86b377SEli Cohen void *in; 12431a86b377SEli Cohen int i, j; 12441a86b377SEli Cohen int err; 12451a86b377SEli Cohen 124652893733SEli Cohen max_rqt = min_t(int, MLX5_MAX_SUPPORTED_VQS / 2, 124752893733SEli Cohen 1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size)); 124852893733SEli Cohen if (max_rqt < 1) 12491a86b377SEli Cohen return -EOPNOTSUPP; 12501a86b377SEli Cohen 125152893733SEli Cohen inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + max_rqt * MLX5_ST_SZ_BYTES(rq_num); 12521a86b377SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 12531a86b377SEli Cohen if (!in) 12541a86b377SEli Cohen return -ENOMEM; 12551a86b377SEli Cohen 12561a86b377SEli Cohen MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); 12571a86b377SEli Cohen rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 12581a86b377SEli Cohen 12591a86b377SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 126052893733SEli Cohen MLX5_SET(rqtc, rqtc, rqt_max_size, max_rqt); 12611a86b377SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1262*a7f46ba4SEli Cohen for (i = 0, j = 0; i < max_rqt; i++, j += 2) 1263*a7f46ba4SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j % ndev->mvdev.max_vqs].virtq_id); 12641a86b377SEli Cohen 1265*a7f46ba4SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, max_rqt); 12661a86b377SEli Cohen err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); 12671a86b377SEli Cohen kfree(in); 12681a86b377SEli Cohen if (err) 12691a86b377SEli Cohen return err; 12701a86b377SEli Cohen 12711a86b377SEli Cohen return 0; 12721a86b377SEli Cohen } 12731a86b377SEli Cohen 127452893733SEli Cohen #define MLX5_MODIFY_RQT_NUM_RQS ((u64)1) 127552893733SEli Cohen 127652893733SEli Cohen static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) 127752893733SEli Cohen { 127852893733SEli Cohen __be32 *list; 127952893733SEli Cohen int max_rqt; 128052893733SEli Cohen void *rqtc; 128152893733SEli Cohen int inlen; 128252893733SEli Cohen void *in; 128352893733SEli Cohen int i, j; 128452893733SEli Cohen int err; 128552893733SEli Cohen 1286*a7f46ba4SEli Cohen max_rqt = min_t(int, roundup_pow_of_two(ndev->cur_num_vqs / 2), 128752893733SEli Cohen 1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size)); 128852893733SEli Cohen if (max_rqt < 1) 128952893733SEli Cohen return -EOPNOTSUPP; 129052893733SEli Cohen 129152893733SEli Cohen inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + max_rqt * MLX5_ST_SZ_BYTES(rq_num); 129252893733SEli Cohen in = kzalloc(inlen, GFP_KERNEL); 129352893733SEli Cohen if (!in) 129452893733SEli Cohen return -ENOMEM; 129552893733SEli Cohen 129652893733SEli Cohen MLX5_SET(modify_rqt_in, in, uid, ndev->mvdev.res.uid); 129752893733SEli Cohen MLX5_SET64(modify_rqt_in, in, bitmask, MLX5_MODIFY_RQT_NUM_RQS); 129852893733SEli Cohen rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 129952893733SEli Cohen MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 130052893733SEli Cohen 130152893733SEli Cohen list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1302*a7f46ba4SEli Cohen for (i = 0, j = 0; i < max_rqt; i++, j += 2) 1303*a7f46ba4SEli Cohen list[i] = cpu_to_be32(ndev->vqs[j % num].virtq_id); 130452893733SEli Cohen 1305*a7f46ba4SEli Cohen MLX5_SET(rqtc, rqtc, rqt_actual_size, max_rqt); 130652893733SEli Cohen err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn); 130752893733SEli Cohen kfree(in); 130852893733SEli Cohen if (err) 130952893733SEli Cohen return err; 131052893733SEli Cohen 131152893733SEli Cohen return 0; 131252893733SEli Cohen } 131352893733SEli Cohen 13141a86b377SEli Cohen static void destroy_rqt(struct mlx5_vdpa_net *ndev) 13151a86b377SEli Cohen { 13161a86b377SEli Cohen mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); 13171a86b377SEli Cohen } 13181a86b377SEli Cohen 13191a86b377SEli Cohen static int create_tir(struct mlx5_vdpa_net *ndev) 13201a86b377SEli Cohen { 13211a86b377SEli Cohen #define HASH_IP_L4PORTS \ 13221a86b377SEli Cohen (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \ 13231a86b377SEli Cohen MLX5_HASH_FIELD_SEL_L4_DPORT) 13241a86b377SEli Cohen static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 13251a86b377SEli Cohen 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 13261a86b377SEli Cohen 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 13271a86b377SEli Cohen 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 13281a86b377SEli Cohen 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a }; 13291a86b377SEli Cohen void *rss_key; 13301a86b377SEli Cohen void *outer; 13311a86b377SEli Cohen void *tirc; 13321a86b377SEli Cohen void *in; 13331a86b377SEli Cohen int err; 13341a86b377SEli Cohen 13351a86b377SEli Cohen in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL); 13361a86b377SEli Cohen if (!in) 13371a86b377SEli Cohen return -ENOMEM; 13381a86b377SEli Cohen 13391a86b377SEli Cohen MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); 13401a86b377SEli Cohen tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 13411a86b377SEli Cohen MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 13421a86b377SEli Cohen 13431a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 13441a86b377SEli Cohen MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ); 13451a86b377SEli Cohen rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 13461a86b377SEli Cohen memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key)); 13471a86b377SEli Cohen 13481a86b377SEli Cohen outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 13491a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); 13501a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); 13511a86b377SEli Cohen MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS); 13521a86b377SEli Cohen 13531a86b377SEli Cohen MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); 13541a86b377SEli Cohen MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); 13551a86b377SEli Cohen 13561a86b377SEli Cohen err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); 13571a86b377SEli Cohen kfree(in); 13581a86b377SEli Cohen return err; 13591a86b377SEli Cohen } 13601a86b377SEli Cohen 13611a86b377SEli Cohen static void destroy_tir(struct mlx5_vdpa_net *ndev) 13621a86b377SEli Cohen { 13631a86b377SEli Cohen mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); 13641a86b377SEli Cohen } 13651a86b377SEli Cohen 13661a86b377SEli Cohen static int add_fwd_to_tir(struct mlx5_vdpa_net *ndev) 13671a86b377SEli Cohen { 13681a86b377SEli Cohen struct mlx5_flow_destination dest[2] = {}; 13691a86b377SEli Cohen struct mlx5_flow_table_attr ft_attr = {}; 13701a86b377SEli Cohen struct mlx5_flow_act flow_act = {}; 13711a86b377SEli Cohen struct mlx5_flow_namespace *ns; 1372540061acSEli Cohen struct mlx5_flow_spec *spec; 1373540061acSEli Cohen void *headers_c; 1374540061acSEli Cohen void *headers_v; 1375540061acSEli Cohen u8 *dmac_c; 1376540061acSEli Cohen u8 *dmac_v; 13771a86b377SEli Cohen int err; 13781a86b377SEli Cohen 1379540061acSEli Cohen spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1380540061acSEli Cohen if (!spec) 1381540061acSEli Cohen return -ENOMEM; 1382540061acSEli Cohen 1383540061acSEli Cohen spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1384540061acSEli Cohen ft_attr.max_fte = 2; 1385540061acSEli Cohen ft_attr.autogroup.max_num_groups = 2; 13861a86b377SEli Cohen 13871a86b377SEli Cohen ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); 13881a86b377SEli Cohen if (!ns) { 1389540061acSEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "failed to get flow namespace\n"); 1390540061acSEli Cohen err = -EOPNOTSUPP; 1391540061acSEli Cohen goto err_ns; 13921a86b377SEli Cohen } 13931a86b377SEli Cohen 13941a86b377SEli Cohen ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 1395540061acSEli Cohen if (IS_ERR(ndev->rxft)) { 1396540061acSEli Cohen err = PTR_ERR(ndev->rxft); 1397540061acSEli Cohen goto err_ns; 1398540061acSEli Cohen } 13991a86b377SEli Cohen 14001a86b377SEli Cohen ndev->rx_counter = mlx5_fc_create(ndev->mvdev.mdev, false); 14011a86b377SEli Cohen if (IS_ERR(ndev->rx_counter)) { 14021a86b377SEli Cohen err = PTR_ERR(ndev->rx_counter); 14031a86b377SEli Cohen goto err_fc; 14041a86b377SEli Cohen } 14051a86b377SEli Cohen 1406540061acSEli Cohen headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers); 1407540061acSEli Cohen dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, outer_headers.dmac_47_16); 1408540061acSEli Cohen memset(dmac_c, 0xff, ETH_ALEN); 1409540061acSEli Cohen headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); 1410540061acSEli Cohen dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16); 1411540061acSEli Cohen ether_addr_copy(dmac_v, ndev->config.mac); 1412540061acSEli Cohen 14131a86b377SEli Cohen flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_COUNT; 14141a86b377SEli Cohen dest[0].type = MLX5_FLOW_DESTINATION_TYPE_TIR; 14151a86b377SEli Cohen dest[0].tir_num = ndev->res.tirn; 14161a86b377SEli Cohen dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 14171a86b377SEli Cohen dest[1].counter_id = mlx5_fc_id(ndev->rx_counter); 1418540061acSEli Cohen ndev->rx_rule_ucast = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dest, 2); 1419540061acSEli Cohen 1420540061acSEli Cohen if (IS_ERR(ndev->rx_rule_ucast)) { 1421540061acSEli Cohen err = PTR_ERR(ndev->rx_rule_ucast); 1422540061acSEli Cohen ndev->rx_rule_ucast = NULL; 1423540061acSEli Cohen goto err_rule_ucast; 14241a86b377SEli Cohen } 14251a86b377SEli Cohen 1426540061acSEli Cohen memset(dmac_c, 0, ETH_ALEN); 1427540061acSEli Cohen memset(dmac_v, 0, ETH_ALEN); 1428540061acSEli Cohen dmac_c[0] = 1; 1429540061acSEli Cohen dmac_v[0] = 1; 1430540061acSEli Cohen flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1431540061acSEli Cohen ndev->rx_rule_mcast = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dest, 1); 1432540061acSEli Cohen if (IS_ERR(ndev->rx_rule_mcast)) { 1433540061acSEli Cohen err = PTR_ERR(ndev->rx_rule_mcast); 1434540061acSEli Cohen ndev->rx_rule_mcast = NULL; 1435540061acSEli Cohen goto err_rule_mcast; 1436540061acSEli Cohen } 1437540061acSEli Cohen 1438540061acSEli Cohen kvfree(spec); 14391a86b377SEli Cohen return 0; 14401a86b377SEli Cohen 1441540061acSEli Cohen err_rule_mcast: 1442540061acSEli Cohen mlx5_del_flow_rules(ndev->rx_rule_ucast); 1443540061acSEli Cohen ndev->rx_rule_ucast = NULL; 1444540061acSEli Cohen err_rule_ucast: 14451a86b377SEli Cohen mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter); 14461a86b377SEli Cohen err_fc: 14471a86b377SEli Cohen mlx5_destroy_flow_table(ndev->rxft); 1448540061acSEli Cohen err_ns: 1449540061acSEli Cohen kvfree(spec); 14501a86b377SEli Cohen return err; 14511a86b377SEli Cohen } 14521a86b377SEli Cohen 14531a86b377SEli Cohen static void remove_fwd_to_tir(struct mlx5_vdpa_net *ndev) 14541a86b377SEli Cohen { 1455540061acSEli Cohen if (!ndev->rx_rule_ucast) 14561a86b377SEli Cohen return; 14571a86b377SEli Cohen 1458540061acSEli Cohen mlx5_del_flow_rules(ndev->rx_rule_mcast); 1459540061acSEli Cohen ndev->rx_rule_mcast = NULL; 1460540061acSEli Cohen mlx5_del_flow_rules(ndev->rx_rule_ucast); 1461540061acSEli Cohen ndev->rx_rule_ucast = NULL; 14621a86b377SEli Cohen mlx5_fc_destroy(ndev->mvdev.mdev, ndev->rx_counter); 14631a86b377SEli Cohen mlx5_destroy_flow_table(ndev->rxft); 14641a86b377SEli Cohen } 14651a86b377SEli Cohen 14665262912eSEli Cohen static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) 14675262912eSEli Cohen { 14685262912eSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 14695262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 14705262912eSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 14715262912eSEli Cohen struct mlx5_core_dev *pfmdev; 14725262912eSEli Cohen size_t read; 14735262912eSEli Cohen u8 mac[ETH_ALEN]; 14745262912eSEli Cohen 14755262912eSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 14765262912eSEli Cohen switch (cmd) { 14775262912eSEli Cohen case VIRTIO_NET_CTRL_MAC_ADDR_SET: 14785262912eSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)mac, ETH_ALEN); 14795262912eSEli Cohen if (read != ETH_ALEN) 14805262912eSEli Cohen break; 14815262912eSEli Cohen 14825262912eSEli Cohen if (!memcmp(ndev->config.mac, mac, 6)) { 14835262912eSEli Cohen status = VIRTIO_NET_OK; 14845262912eSEli Cohen break; 14855262912eSEli Cohen } 14865262912eSEli Cohen 14875262912eSEli Cohen if (!is_zero_ether_addr(ndev->config.mac)) { 14885262912eSEli Cohen if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 14895262912eSEli Cohen mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n", 14905262912eSEli Cohen ndev->config.mac); 14915262912eSEli Cohen break; 14925262912eSEli Cohen } 14935262912eSEli Cohen } 14945262912eSEli Cohen 14955262912eSEli Cohen if (mlx5_mpfs_add_mac(pfmdev, mac)) { 14965262912eSEli Cohen mlx5_vdpa_warn(mvdev, "failed to insert new MAC %pM into MPFS table\n", 14975262912eSEli Cohen mac); 14985262912eSEli Cohen break; 14995262912eSEli Cohen } 15005262912eSEli Cohen 15015262912eSEli Cohen memcpy(ndev->config.mac, mac, ETH_ALEN); 15025262912eSEli Cohen status = VIRTIO_NET_OK; 15035262912eSEli Cohen break; 15045262912eSEli Cohen 15055262912eSEli Cohen default: 15065262912eSEli Cohen break; 15075262912eSEli Cohen } 15085262912eSEli Cohen 15095262912eSEli Cohen return status; 15105262912eSEli Cohen } 15115262912eSEli Cohen 151252893733SEli Cohen static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps) 151352893733SEli Cohen { 151452893733SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 151552893733SEli Cohen int cur_qps = ndev->cur_num_vqs / 2; 151652893733SEli Cohen int err; 151752893733SEli Cohen int i; 151852893733SEli Cohen 151952893733SEli Cohen if (cur_qps > newqps) { 152052893733SEli Cohen err = modify_rqt(ndev, 2 * newqps); 152152893733SEli Cohen if (err) 152252893733SEli Cohen return err; 152352893733SEli Cohen 152452893733SEli Cohen for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) 152552893733SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 152652893733SEli Cohen 152752893733SEli Cohen ndev->cur_num_vqs = 2 * newqps; 152852893733SEli Cohen } else { 152952893733SEli Cohen ndev->cur_num_vqs = 2 * newqps; 153052893733SEli Cohen for (i = cur_qps * 2; i < 2 * newqps; i++) { 153152893733SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 153252893733SEli Cohen if (err) 153352893733SEli Cohen goto clean_added; 153452893733SEli Cohen } 153552893733SEli Cohen err = modify_rqt(ndev, 2 * newqps); 153652893733SEli Cohen if (err) 153752893733SEli Cohen goto clean_added; 153852893733SEli Cohen } 153952893733SEli Cohen return 0; 154052893733SEli Cohen 154152893733SEli Cohen clean_added: 154252893733SEli Cohen for (--i; i >= cur_qps; --i) 154352893733SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 154452893733SEli Cohen 154552893733SEli Cohen return err; 154652893733SEli Cohen } 154752893733SEli Cohen 154852893733SEli Cohen static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd) 154952893733SEli Cohen { 155052893733SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 155152893733SEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 155252893733SEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 155352893733SEli Cohen struct virtio_net_ctrl_mq mq; 155452893733SEli Cohen size_t read; 155552893733SEli Cohen u16 newqps; 155652893733SEli Cohen 155752893733SEli Cohen switch (cmd) { 155852893733SEli Cohen case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: 155952893733SEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)&mq, sizeof(mq)); 156052893733SEli Cohen if (read != sizeof(mq)) 156152893733SEli Cohen break; 156252893733SEli Cohen 156352893733SEli Cohen newqps = mlx5vdpa16_to_cpu(mvdev, mq.virtqueue_pairs); 156452893733SEli Cohen if (ndev->cur_num_vqs == 2 * newqps) { 156552893733SEli Cohen status = VIRTIO_NET_OK; 156652893733SEli Cohen break; 156752893733SEli Cohen } 156852893733SEli Cohen 156952893733SEli Cohen if (!change_num_qps(mvdev, newqps)) 157052893733SEli Cohen status = VIRTIO_NET_OK; 157152893733SEli Cohen 157252893733SEli Cohen break; 157352893733SEli Cohen default: 157452893733SEli Cohen break; 157552893733SEli Cohen } 157652893733SEli Cohen 157752893733SEli Cohen return status; 157852893733SEli Cohen } 157952893733SEli Cohen 15805262912eSEli Cohen static void mlx5_cvq_kick_handler(struct work_struct *work) 15815262912eSEli Cohen { 15825262912eSEli Cohen virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 15835262912eSEli Cohen struct virtio_net_ctrl_hdr ctrl; 1584218bdd20SEli Cohen struct mlx5_vdpa_wq_ent *wqent; 15855262912eSEli Cohen struct mlx5_vdpa_dev *mvdev; 15865262912eSEli Cohen struct mlx5_control_vq *cvq; 15875262912eSEli Cohen struct mlx5_vdpa_net *ndev; 15885262912eSEli Cohen size_t read, write; 15895262912eSEli Cohen int err; 15905262912eSEli Cohen 1591218bdd20SEli Cohen wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 15925262912eSEli Cohen mvdev = wqent->mvdev; 15935262912eSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 15945262912eSEli Cohen cvq = &mvdev->cvq; 15955262912eSEli Cohen if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 15965262912eSEli Cohen goto out; 15975262912eSEli Cohen 15985262912eSEli Cohen if (!cvq->ready) 15995262912eSEli Cohen goto out; 16005262912eSEli Cohen 16015262912eSEli Cohen while (true) { 16025262912eSEli Cohen err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head, 16035262912eSEli Cohen GFP_ATOMIC); 16045262912eSEli Cohen if (err <= 0) 16055262912eSEli Cohen break; 16065262912eSEli Cohen 16075262912eSEli Cohen read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &ctrl, sizeof(ctrl)); 16085262912eSEli Cohen if (read != sizeof(ctrl)) 16095262912eSEli Cohen break; 16105262912eSEli Cohen 16115262912eSEli Cohen switch (ctrl.class) { 16125262912eSEli Cohen case VIRTIO_NET_CTRL_MAC: 16135262912eSEli Cohen status = handle_ctrl_mac(mvdev, ctrl.cmd); 16145262912eSEli Cohen break; 161552893733SEli Cohen case VIRTIO_NET_CTRL_MQ: 161652893733SEli Cohen status = handle_ctrl_mq(mvdev, ctrl.cmd); 161752893733SEli Cohen break; 16185262912eSEli Cohen 16195262912eSEli Cohen default: 16205262912eSEli Cohen break; 16215262912eSEli Cohen } 16225262912eSEli Cohen 16235262912eSEli Cohen /* Make sure data is written before advancing index */ 16245262912eSEli Cohen smp_wmb(); 16255262912eSEli Cohen 16265262912eSEli Cohen write = vringh_iov_push_iotlb(&cvq->vring, &cvq->wiov, &status, sizeof(status)); 16275262912eSEli Cohen vringh_complete_iotlb(&cvq->vring, cvq->head, write); 16285262912eSEli Cohen vringh_kiov_cleanup(&cvq->riov); 16295262912eSEli Cohen vringh_kiov_cleanup(&cvq->wiov); 16305262912eSEli Cohen 16315262912eSEli Cohen if (vringh_need_notify_iotlb(&cvq->vring)) 16325262912eSEli Cohen vringh_notify(&cvq->vring); 16335262912eSEli Cohen } 16345262912eSEli Cohen out: 16355262912eSEli Cohen kfree(wqent); 16365262912eSEli Cohen } 16375262912eSEli Cohen 16381a86b377SEli Cohen static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) 16391a86b377SEli Cohen { 16401a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 16411a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 16425262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 1643218bdd20SEli Cohen struct mlx5_vdpa_wq_ent *wqent; 16441a86b377SEli Cohen 1645e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1646e4fc6650SEli Cohen return; 1647e4fc6650SEli Cohen 16485262912eSEli Cohen if (unlikely(is_ctrl_vq_idx(mvdev, idx))) { 16495262912eSEli Cohen if (!mvdev->cvq.ready) 16505262912eSEli Cohen return; 16515262912eSEli Cohen 16525262912eSEli Cohen wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); 16535262912eSEli Cohen if (!wqent) 16545262912eSEli Cohen return; 16555262912eSEli Cohen 16565262912eSEli Cohen wqent->mvdev = mvdev; 16575262912eSEli Cohen INIT_WORK(&wqent->work, mlx5_cvq_kick_handler); 16585262912eSEli Cohen queue_work(mvdev->wq, &wqent->work); 16595262912eSEli Cohen return; 16605262912eSEli Cohen } 16615262912eSEli Cohen 16625262912eSEli Cohen mvq = &ndev->vqs[idx]; 16631a86b377SEli Cohen if (unlikely(!mvq->ready)) 16641a86b377SEli Cohen return; 16651a86b377SEli Cohen 16661a86b377SEli Cohen iowrite16(idx, ndev->mvdev.res.kick_addr); 16671a86b377SEli Cohen } 16681a86b377SEli Cohen 16691a86b377SEli Cohen static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 16701a86b377SEli Cohen u64 driver_area, u64 device_area) 16711a86b377SEli Cohen { 16721a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 16731a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 16745262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 16751a86b377SEli Cohen 1676e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1677e4fc6650SEli Cohen return -EINVAL; 1678e4fc6650SEli Cohen 16795262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 16805262912eSEli Cohen mvdev->cvq.desc_addr = desc_area; 16815262912eSEli Cohen mvdev->cvq.device_addr = device_area; 16825262912eSEli Cohen mvdev->cvq.driver_addr = driver_area; 16835262912eSEli Cohen return 0; 16845262912eSEli Cohen } 16855262912eSEli Cohen 16865262912eSEli Cohen mvq = &ndev->vqs[idx]; 16871a86b377SEli Cohen mvq->desc_addr = desc_area; 16881a86b377SEli Cohen mvq->device_addr = device_area; 16891a86b377SEli Cohen mvq->driver_addr = driver_area; 16901a86b377SEli Cohen return 0; 16911a86b377SEli Cohen } 16921a86b377SEli Cohen 16931a86b377SEli Cohen static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 16941a86b377SEli Cohen { 16951a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 16961a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 16971a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 16981a86b377SEli Cohen 16995262912eSEli Cohen if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 1700e4fc6650SEli Cohen return; 1701e4fc6650SEli Cohen 17021a86b377SEli Cohen mvq = &ndev->vqs[idx]; 17031a86b377SEli Cohen mvq->num_ent = num; 17041a86b377SEli Cohen } 17051a86b377SEli Cohen 17061a86b377SEli Cohen static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 17071a86b377SEli Cohen { 17081a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17091a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17101a86b377SEli Cohen 1711db296d25SEli Cohen ndev->event_cbs[idx] = *cb; 17121a86b377SEli Cohen } 17131a86b377SEli Cohen 17145262912eSEli Cohen static void mlx5_cvq_notify(struct vringh *vring) 17155262912eSEli Cohen { 17165262912eSEli Cohen struct mlx5_control_vq *cvq = container_of(vring, struct mlx5_control_vq, vring); 17175262912eSEli Cohen 17185262912eSEli Cohen if (!cvq->event_cb.callback) 17195262912eSEli Cohen return; 17205262912eSEli Cohen 17215262912eSEli Cohen cvq->event_cb.callback(cvq->event_cb.private); 17225262912eSEli Cohen } 17235262912eSEli Cohen 17245262912eSEli Cohen static void set_cvq_ready(struct mlx5_vdpa_dev *mvdev, bool ready) 17255262912eSEli Cohen { 17265262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 17275262912eSEli Cohen 17285262912eSEli Cohen cvq->ready = ready; 17295262912eSEli Cohen if (!ready) 17305262912eSEli Cohen return; 17315262912eSEli Cohen 17325262912eSEli Cohen cvq->vring.notify = mlx5_cvq_notify; 17331a86b377SEli Cohen } 17341a86b377SEli Cohen 17351a86b377SEli Cohen static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 17361a86b377SEli Cohen { 17371a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17381a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17395262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 17401a86b377SEli Cohen 1741759be899SEli Cohen if (!mvdev->actual_features) 1742759be899SEli Cohen return; 1743759be899SEli Cohen 1744e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1745e4fc6650SEli Cohen return; 1746e4fc6650SEli Cohen 17475262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 17485262912eSEli Cohen set_cvq_ready(mvdev, ready); 17495262912eSEli Cohen return; 17505262912eSEli Cohen } 17515262912eSEli Cohen 17525262912eSEli Cohen mvq = &ndev->vqs[idx]; 17531a86b377SEli Cohen if (!ready) 17541a86b377SEli Cohen suspend_vq(ndev, mvq); 17551a86b377SEli Cohen 17561a86b377SEli Cohen mvq->ready = ready; 17571a86b377SEli Cohen } 17581a86b377SEli Cohen 17591a86b377SEli Cohen static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx) 17601a86b377SEli Cohen { 17611a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17621a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17631a86b377SEli Cohen 1764e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1765e4fc6650SEli Cohen return false; 1766e4fc6650SEli Cohen 17675262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) 17685262912eSEli Cohen return mvdev->cvq.ready; 17695262912eSEli Cohen 17705262912eSEli Cohen return ndev->vqs[idx].ready; 17711a86b377SEli Cohen } 17721a86b377SEli Cohen 17731a86b377SEli Cohen static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx, 17741a86b377SEli Cohen const struct vdpa_vq_state *state) 17751a86b377SEli Cohen { 17761a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 17771a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 17785262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 17791a86b377SEli Cohen 1780e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1781e4fc6650SEli Cohen return -EINVAL; 1782e4fc6650SEli Cohen 17835262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 17845262912eSEli Cohen mvdev->cvq.vring.last_avail_idx = state->split.avail_index; 17855262912eSEli Cohen return 0; 17865262912eSEli Cohen } 17875262912eSEli Cohen 17885262912eSEli Cohen mvq = &ndev->vqs[idx]; 17891a86b377SEli Cohen if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) { 17901a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "can't modify available index\n"); 17911a86b377SEli Cohen return -EINVAL; 17921a86b377SEli Cohen } 17931a86b377SEli Cohen 1794530a5678SJason Wang mvq->used_idx = state->split.avail_index; 1795530a5678SJason Wang mvq->avail_idx = state->split.avail_index; 17961a86b377SEli Cohen return 0; 17971a86b377SEli Cohen } 17981a86b377SEli Cohen 17991a86b377SEli Cohen static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 18001a86b377SEli Cohen { 18011a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 18021a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 18035262912eSEli Cohen struct mlx5_vdpa_virtqueue *mvq; 18041a86b377SEli Cohen struct mlx5_virtq_attr attr; 18051a86b377SEli Cohen int err; 18061a86b377SEli Cohen 1807e4fc6650SEli Cohen if (!is_index_valid(mvdev, idx)) 1808e4fc6650SEli Cohen return -EINVAL; 1809e4fc6650SEli Cohen 18105262912eSEli Cohen if (is_ctrl_vq_idx(mvdev, idx)) { 18115262912eSEli Cohen state->split.avail_index = mvdev->cvq.vring.last_avail_idx; 18125262912eSEli Cohen return 0; 18135262912eSEli Cohen } 18145262912eSEli Cohen 18155262912eSEli Cohen mvq = &ndev->vqs[idx]; 18163176e974SSi-Wei Liu /* If the virtq object was destroyed, use the value saved at 18173176e974SSi-Wei Liu * the last minute of suspend_vq. This caters for userspace 18183176e974SSi-Wei Liu * that cares about emulating the index after vq is stopped. 18193176e974SSi-Wei Liu */ 18203176e974SSi-Wei Liu if (!mvq->initialized) { 1821bc04d93eSEli Cohen /* Firmware returns a wrong value for the available index. 1822bc04d93eSEli Cohen * Since both values should be identical, we take the value of 1823bc04d93eSEli Cohen * used_idx which is reported correctly. 1824bc04d93eSEli Cohen */ 1825530a5678SJason Wang state->split.avail_index = mvq->used_idx; 18263176e974SSi-Wei Liu return 0; 18273176e974SSi-Wei Liu } 18281a86b377SEli Cohen 18291a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 18301a86b377SEli Cohen if (err) { 18311a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n"); 18321a86b377SEli Cohen return err; 18331a86b377SEli Cohen } 1834530a5678SJason Wang state->split.avail_index = attr.used_index; 18351a86b377SEli Cohen return 0; 18361a86b377SEli Cohen } 18371a86b377SEli Cohen 18381a86b377SEli Cohen static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) 18391a86b377SEli Cohen { 18401a86b377SEli Cohen return PAGE_SIZE; 18411a86b377SEli Cohen } 18421a86b377SEli Cohen 18431a86b377SEli Cohen enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9, 18441a86b377SEli Cohen MLX5_VIRTIO_NET_F_CSUM = 1 << 10, 18451a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11, 18461a86b377SEli Cohen MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12, 18471a86b377SEli Cohen }; 18481a86b377SEli Cohen 18491a86b377SEli Cohen static u64 mlx_to_vritio_features(u16 dev_features) 18501a86b377SEli Cohen { 18511a86b377SEli Cohen u64 result = 0; 18521a86b377SEli Cohen 18531a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM) 1854cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM); 18551a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_CSUM) 1856cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_CSUM); 18571a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6) 1858cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6); 18591a86b377SEli Cohen if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4) 1860cbb52359SNathan Chancellor result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4); 18611a86b377SEli Cohen 18621a86b377SEli Cohen return result; 18631a86b377SEli Cohen } 18641a86b377SEli Cohen 1865a64917bcSEli Cohen static u64 mlx5_vdpa_get_device_features(struct vdpa_device *vdev) 18661a86b377SEli Cohen { 18671a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 18681a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 18691a86b377SEli Cohen u16 dev_features; 18701a86b377SEli Cohen 18711a86b377SEli Cohen dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, device_features_bits_mask); 18725262912eSEli Cohen ndev->mvdev.mlx_features |= mlx_to_vritio_features(dev_features); 18731a86b377SEli Cohen if (MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, virtio_version_1_0)) 1874cbb52359SNathan Chancellor ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_F_VERSION_1); 1875cbb52359SNathan Chancellor ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM); 18765262912eSEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VQ); 18775262912eSEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR); 187852893733SEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MQ); 1879edf747afSEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_STATUS); 188060af39c1SEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MTU); 18815262912eSEli Cohen 18821a86b377SEli Cohen print_features(mvdev, ndev->mvdev.mlx_features, false); 18831a86b377SEli Cohen return ndev->mvdev.mlx_features; 18841a86b377SEli Cohen } 18851a86b377SEli Cohen 18861a86b377SEli Cohen static int verify_min_features(struct mlx5_vdpa_dev *mvdev, u64 features) 18871a86b377SEli Cohen { 1888cbb52359SNathan Chancellor if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) 18891a86b377SEli Cohen return -EOPNOTSUPP; 18901a86b377SEli Cohen 18911a86b377SEli Cohen return 0; 18921a86b377SEli Cohen } 18931a86b377SEli Cohen 1894ae0428deSEli Cohen static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev) 18951a86b377SEli Cohen { 1896ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 18975262912eSEli Cohen struct mlx5_control_vq *cvq = &mvdev->cvq; 18981a86b377SEli Cohen int err; 18991a86b377SEli Cohen int i; 19001a86b377SEli Cohen 1901ae0428deSEli Cohen for (i = 0; i < 2 * mlx5_vdpa_max_qps(mvdev->max_vqs); i++) { 19021a86b377SEli Cohen err = setup_vq(ndev, &ndev->vqs[i]); 19031a86b377SEli Cohen if (err) 19041a86b377SEli Cohen goto err_vq; 19051a86b377SEli Cohen } 19061a86b377SEli Cohen 19075262912eSEli Cohen if (mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) { 19085262912eSEli Cohen err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features, 19095262912eSEli Cohen MLX5_CVQ_MAX_ENT, false, 19105262912eSEli Cohen (struct vring_desc *)(uintptr_t)cvq->desc_addr, 19115262912eSEli Cohen (struct vring_avail *)(uintptr_t)cvq->driver_addr, 19125262912eSEli Cohen (struct vring_used *)(uintptr_t)cvq->device_addr); 19135262912eSEli Cohen if (err) 19145262912eSEli Cohen goto err_vq; 19155262912eSEli Cohen } 19165262912eSEli Cohen 19171a86b377SEli Cohen return 0; 19181a86b377SEli Cohen 19191a86b377SEli Cohen err_vq: 19201a86b377SEli Cohen for (--i; i >= 0; i--) 19211a86b377SEli Cohen teardown_vq(ndev, &ndev->vqs[i]); 19221a86b377SEli Cohen 19231a86b377SEli Cohen return err; 19241a86b377SEli Cohen } 19251a86b377SEli Cohen 19261a86b377SEli Cohen static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) 19271a86b377SEli Cohen { 19281a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 19291a86b377SEli Cohen int i; 19301a86b377SEli Cohen 19311a86b377SEli Cohen for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { 19321a86b377SEli Cohen mvq = &ndev->vqs[i]; 19331a86b377SEli Cohen if (!mvq->initialized) 19341a86b377SEli Cohen continue; 19351a86b377SEli Cohen 19361a86b377SEli Cohen teardown_vq(ndev, mvq); 19371a86b377SEli Cohen } 19381a86b377SEli Cohen } 19391a86b377SEli Cohen 1940e4fc6650SEli Cohen static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) 19418a7c3213SMichael S. Tsirkin { 1942e4fc6650SEli Cohen if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_CTRL_VQ)) { 1943e4fc6650SEli Cohen if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) { 1944e4fc6650SEli Cohen /* MQ supported. CVQ index is right above the last data virtqueue's */ 1945e4fc6650SEli Cohen mvdev->max_idx = mvdev->max_vqs; 1946e4fc6650SEli Cohen } else { 1947e4fc6650SEli Cohen /* Only CVQ supportted. data virtqueues occupy indices 0 and 1. 1948e4fc6650SEli Cohen * CVQ gets index 2 1949e4fc6650SEli Cohen */ 1950e4fc6650SEli Cohen mvdev->max_idx = 2; 19518a7c3213SMichael S. Tsirkin } 1952e4fc6650SEli Cohen } else { 1953e4fc6650SEli Cohen /* Two data virtqueues only: one for rx and one for tx */ 1954e4fc6650SEli Cohen mvdev->max_idx = 1; 1955e4fc6650SEli Cohen } 195636bdcf31SEli Cohen } 195736bdcf31SEli Cohen 1958a64917bcSEli Cohen static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) 19591a86b377SEli Cohen { 19601a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 19611a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 19621a86b377SEli Cohen int err; 19631a86b377SEli Cohen 19641a86b377SEli Cohen print_features(mvdev, features, true); 19651a86b377SEli Cohen 19661a86b377SEli Cohen err = verify_min_features(mvdev, features); 19671a86b377SEli Cohen if (err) 19681a86b377SEli Cohen return err; 19691a86b377SEli Cohen 19701a86b377SEli Cohen ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; 1971e4fc6650SEli Cohen update_cvq_info(mvdev); 19721a86b377SEli Cohen return err; 19731a86b377SEli Cohen } 19741a86b377SEli Cohen 19751a86b377SEli Cohen static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 19761a86b377SEli Cohen { 1977edf747afSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 1978edf747afSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1979edf747afSEli Cohen 1980edf747afSEli Cohen ndev->config_cb = *cb; 19811a86b377SEli Cohen } 19821a86b377SEli Cohen 19831a86b377SEli Cohen #define MLX5_VDPA_MAX_VQ_ENTRIES 256 19841a86b377SEli Cohen static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev) 19851a86b377SEli Cohen { 19861a86b377SEli Cohen return MLX5_VDPA_MAX_VQ_ENTRIES; 19871a86b377SEli Cohen } 19881a86b377SEli Cohen 19891a86b377SEli Cohen static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev) 19901a86b377SEli Cohen { 19911a86b377SEli Cohen return VIRTIO_ID_NET; 19921a86b377SEli Cohen } 19931a86b377SEli Cohen 19941a86b377SEli Cohen static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev) 19951a86b377SEli Cohen { 19961a86b377SEli Cohen return PCI_VENDOR_ID_MELLANOX; 19971a86b377SEli Cohen } 19981a86b377SEli Cohen 19991a86b377SEli Cohen static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev) 20001a86b377SEli Cohen { 20011a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 20021a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20031a86b377SEli Cohen 20041a86b377SEli Cohen print_status(mvdev, ndev->mvdev.status, false); 20051a86b377SEli Cohen return ndev->mvdev.status; 20061a86b377SEli Cohen } 20071a86b377SEli Cohen 20081a86b377SEli Cohen static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 20091a86b377SEli Cohen { 20101a86b377SEli Cohen struct mlx5_vq_restore_info *ri = &mvq->ri; 201152893733SEli Cohen struct mlx5_virtq_attr attr = {}; 20121a86b377SEli Cohen int err; 20131a86b377SEli Cohen 201452893733SEli Cohen if (mvq->initialized) { 20151a86b377SEli Cohen err = query_virtqueue(ndev, mvq, &attr); 20161a86b377SEli Cohen if (err) 20171a86b377SEli Cohen return err; 201852893733SEli Cohen } 20191a86b377SEli Cohen 20201a86b377SEli Cohen ri->avail_index = attr.available_index; 2021b35ccebeSEli Cohen ri->used_index = attr.used_index; 20221a86b377SEli Cohen ri->ready = mvq->ready; 20231a86b377SEli Cohen ri->num_ent = mvq->num_ent; 20241a86b377SEli Cohen ri->desc_addr = mvq->desc_addr; 20251a86b377SEli Cohen ri->device_addr = mvq->device_addr; 20261a86b377SEli Cohen ri->driver_addr = mvq->driver_addr; 20271a86b377SEli Cohen ri->restore = true; 20281a86b377SEli Cohen return 0; 20291a86b377SEli Cohen } 20301a86b377SEli Cohen 20311a86b377SEli Cohen static int save_channels_info(struct mlx5_vdpa_net *ndev) 20321a86b377SEli Cohen { 20331a86b377SEli Cohen int i; 20341a86b377SEli Cohen 20351a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 20361a86b377SEli Cohen memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); 20371a86b377SEli Cohen save_channel_info(ndev, &ndev->vqs[i]); 20381a86b377SEli Cohen } 20391a86b377SEli Cohen return 0; 20401a86b377SEli Cohen } 20411a86b377SEli Cohen 20421a86b377SEli Cohen static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) 20431a86b377SEli Cohen { 20441a86b377SEli Cohen int i; 20451a86b377SEli Cohen 20461a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 20471a86b377SEli Cohen memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 20481a86b377SEli Cohen } 20491a86b377SEli Cohen 20501a86b377SEli Cohen static void restore_channels_info(struct mlx5_vdpa_net *ndev) 20511a86b377SEli Cohen { 20521a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 20531a86b377SEli Cohen struct mlx5_vq_restore_info *ri; 20541a86b377SEli Cohen int i; 20551a86b377SEli Cohen 20561a86b377SEli Cohen mlx5_clear_vqs(ndev); 20571a86b377SEli Cohen init_mvqs(ndev); 20581a86b377SEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) { 20591a86b377SEli Cohen mvq = &ndev->vqs[i]; 20601a86b377SEli Cohen ri = &mvq->ri; 20611a86b377SEli Cohen if (!ri->restore) 20621a86b377SEli Cohen continue; 20631a86b377SEli Cohen 20641a86b377SEli Cohen mvq->avail_idx = ri->avail_index; 2065b35ccebeSEli Cohen mvq->used_idx = ri->used_index; 20661a86b377SEli Cohen mvq->ready = ri->ready; 20671a86b377SEli Cohen mvq->num_ent = ri->num_ent; 20681a86b377SEli Cohen mvq->desc_addr = ri->desc_addr; 20691a86b377SEli Cohen mvq->device_addr = ri->device_addr; 20701a86b377SEli Cohen mvq->driver_addr = ri->driver_addr; 20711a86b377SEli Cohen } 20721a86b377SEli Cohen } 20731a86b377SEli Cohen 2074ae0428deSEli Cohen static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) 20751a86b377SEli Cohen { 2076ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 20771a86b377SEli Cohen int err; 20781a86b377SEli Cohen 20791a86b377SEli Cohen suspend_vqs(ndev); 20801a86b377SEli Cohen err = save_channels_info(ndev); 20811a86b377SEli Cohen if (err) 20821a86b377SEli Cohen goto err_mr; 20831a86b377SEli Cohen 20841a86b377SEli Cohen teardown_driver(ndev); 2085ae0428deSEli Cohen mlx5_vdpa_destroy_mr(mvdev); 2086ae0428deSEli Cohen err = mlx5_vdpa_create_mr(mvdev, iotlb); 20871a86b377SEli Cohen if (err) 20881a86b377SEli Cohen goto err_mr; 20891a86b377SEli Cohen 2090ae0428deSEli Cohen if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 20911897f0b6SEli Cohen return 0; 20921897f0b6SEli Cohen 20931a86b377SEli Cohen restore_channels_info(ndev); 2094ae0428deSEli Cohen err = setup_driver(mvdev); 20951a86b377SEli Cohen if (err) 20961a86b377SEli Cohen goto err_setup; 20971a86b377SEli Cohen 20981a86b377SEli Cohen return 0; 20991a86b377SEli Cohen 21001a86b377SEli Cohen err_setup: 2101ae0428deSEli Cohen mlx5_vdpa_destroy_mr(mvdev); 21021a86b377SEli Cohen err_mr: 21031a86b377SEli Cohen return err; 21041a86b377SEli Cohen } 21051a86b377SEli Cohen 2106ae0428deSEli Cohen static int setup_driver(struct mlx5_vdpa_dev *mvdev) 21071a86b377SEli Cohen { 2108ae0428deSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 21091a86b377SEli Cohen int err; 21101a86b377SEli Cohen 21111a86b377SEli Cohen mutex_lock(&ndev->reslock); 21121a86b377SEli Cohen if (ndev->setup) { 2113ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n"); 21141a86b377SEli Cohen err = 0; 21151a86b377SEli Cohen goto out; 21161a86b377SEli Cohen } 2117ae0428deSEli Cohen err = setup_virtqueues(mvdev); 21181a86b377SEli Cohen if (err) { 2119ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "setup_virtqueues\n"); 21201a86b377SEli Cohen goto out; 21211a86b377SEli Cohen } 21221a86b377SEli Cohen 21231a86b377SEli Cohen err = create_rqt(ndev); 21241a86b377SEli Cohen if (err) { 2125ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "create_rqt\n"); 21261a86b377SEli Cohen goto err_rqt; 21271a86b377SEli Cohen } 21281a86b377SEli Cohen 21291a86b377SEli Cohen err = create_tir(ndev); 21301a86b377SEli Cohen if (err) { 2131ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "create_tir\n"); 21321a86b377SEli Cohen goto err_tir; 21331a86b377SEli Cohen } 21341a86b377SEli Cohen 21351a86b377SEli Cohen err = add_fwd_to_tir(ndev); 21361a86b377SEli Cohen if (err) { 2137ae0428deSEli Cohen mlx5_vdpa_warn(mvdev, "add_fwd_to_tir\n"); 21381a86b377SEli Cohen goto err_fwd; 21391a86b377SEli Cohen } 21401a86b377SEli Cohen ndev->setup = true; 21411a86b377SEli Cohen mutex_unlock(&ndev->reslock); 21421a86b377SEli Cohen 21431a86b377SEli Cohen return 0; 21441a86b377SEli Cohen 21451a86b377SEli Cohen err_fwd: 21461a86b377SEli Cohen destroy_tir(ndev); 21471a86b377SEli Cohen err_tir: 21481a86b377SEli Cohen destroy_rqt(ndev); 21491a86b377SEli Cohen err_rqt: 21501a86b377SEli Cohen teardown_virtqueues(ndev); 21511a86b377SEli Cohen out: 21521a86b377SEli Cohen mutex_unlock(&ndev->reslock); 21531a86b377SEli Cohen return err; 21541a86b377SEli Cohen } 21551a86b377SEli Cohen 21561a86b377SEli Cohen static void teardown_driver(struct mlx5_vdpa_net *ndev) 21571a86b377SEli Cohen { 21581a86b377SEli Cohen mutex_lock(&ndev->reslock); 21591a86b377SEli Cohen if (!ndev->setup) 21601a86b377SEli Cohen goto out; 21611a86b377SEli Cohen 21621a86b377SEli Cohen remove_fwd_to_tir(ndev); 21631a86b377SEli Cohen destroy_tir(ndev); 21641a86b377SEli Cohen destroy_rqt(ndev); 21651a86b377SEli Cohen teardown_virtqueues(ndev); 21661a86b377SEli Cohen ndev->setup = false; 21671a86b377SEli Cohen out: 21681a86b377SEli Cohen mutex_unlock(&ndev->reslock); 21691a86b377SEli Cohen } 21701a86b377SEli Cohen 2171e3aadf2eSEli Cohen static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) 2172e3aadf2eSEli Cohen { 2173e3aadf2eSEli Cohen int i; 2174e3aadf2eSEli Cohen 2175e3aadf2eSEli Cohen for (i = 0; i < ndev->mvdev.max_vqs; i++) 2176e3aadf2eSEli Cohen ndev->vqs[i].ready = false; 2177ef12e4bfSEli Cohen 2178ef12e4bfSEli Cohen ndev->mvdev.cvq.ready = false; 2179e3aadf2eSEli Cohen } 2180e3aadf2eSEli Cohen 21811a86b377SEli Cohen static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) 21821a86b377SEli Cohen { 21831a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 21841a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 21851a86b377SEli Cohen int err; 21861a86b377SEli Cohen 21871a86b377SEli Cohen print_status(mvdev, status, true); 21881a86b377SEli Cohen 21891a86b377SEli Cohen if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { 21901a86b377SEli Cohen if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 2191ae0428deSEli Cohen err = setup_driver(mvdev); 21921a86b377SEli Cohen if (err) { 21931a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); 21941a86b377SEli Cohen goto err_setup; 21951a86b377SEli Cohen } 21961a86b377SEli Cohen } else { 21971a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); 21981a86b377SEli Cohen return; 21991a86b377SEli Cohen } 22001a86b377SEli Cohen } 22011a86b377SEli Cohen 22021a86b377SEli Cohen ndev->mvdev.status = status; 22031a86b377SEli Cohen return; 22041a86b377SEli Cohen 22051a86b377SEli Cohen err_setup: 22061a86b377SEli Cohen mlx5_vdpa_destroy_mr(&ndev->mvdev); 22071a86b377SEli Cohen ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; 22081a86b377SEli Cohen } 22091a86b377SEli Cohen 22100686082dSXie Yongji static int mlx5_vdpa_reset(struct vdpa_device *vdev) 22110686082dSXie Yongji { 22120686082dSXie Yongji struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22130686082dSXie Yongji struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 22140686082dSXie Yongji 22150686082dSXie Yongji print_status(mvdev, 0, true); 22160686082dSXie Yongji mlx5_vdpa_info(mvdev, "performing device reset\n"); 22170686082dSXie Yongji teardown_driver(ndev); 22180686082dSXie Yongji clear_vqs_ready(ndev); 22190686082dSXie Yongji mlx5_vdpa_destroy_mr(&ndev->mvdev); 22200686082dSXie Yongji ndev->mvdev.status = 0; 22210686082dSXie Yongji memset(ndev->event_cbs, 0, sizeof(ndev->event_cbs)); 22220686082dSXie Yongji ndev->mvdev.actual_features = 0; 22230686082dSXie Yongji ++mvdev->generation; 22240686082dSXie Yongji if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 22250686082dSXie Yongji if (mlx5_vdpa_create_mr(mvdev, NULL)) 22260686082dSXie Yongji mlx5_vdpa_warn(mvdev, "create MR failed\n"); 22270686082dSXie Yongji } 22280686082dSXie Yongji 22290686082dSXie Yongji return 0; 22300686082dSXie Yongji } 22310686082dSXie Yongji 2232442706f9SStefano Garzarella static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev) 2233442706f9SStefano Garzarella { 2234442706f9SStefano Garzarella return sizeof(struct virtio_net_config); 2235442706f9SStefano Garzarella } 2236442706f9SStefano Garzarella 22371a86b377SEli Cohen static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, 22381a86b377SEli Cohen unsigned int len) 22391a86b377SEli Cohen { 22401a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22411a86b377SEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 22421a86b377SEli Cohen 2243dcfde163SStefano Garzarella if (offset + len <= sizeof(struct virtio_net_config)) 22442874211fSDan Carpenter memcpy(buf, (u8 *)&ndev->config + offset, len); 22451a86b377SEli Cohen } 22461a86b377SEli Cohen 22471a86b377SEli Cohen static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf, 22481a86b377SEli Cohen unsigned int len) 22491a86b377SEli Cohen { 22501a86b377SEli Cohen /* not supported */ 22511a86b377SEli Cohen } 22521a86b377SEli Cohen 22531a86b377SEli Cohen static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) 22541a86b377SEli Cohen { 22551a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22561a86b377SEli Cohen 22571a86b377SEli Cohen return mvdev->generation; 22581a86b377SEli Cohen } 22591a86b377SEli Cohen 22601a86b377SEli Cohen static int mlx5_vdpa_set_map(struct vdpa_device *vdev, struct vhost_iotlb *iotlb) 22611a86b377SEli Cohen { 22621a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22631a86b377SEli Cohen bool change_map; 22641a86b377SEli Cohen int err; 22651a86b377SEli Cohen 22661a86b377SEli Cohen err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map); 22671a86b377SEli Cohen if (err) { 22681a86b377SEli Cohen mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); 22691a86b377SEli Cohen return err; 22701a86b377SEli Cohen } 22711a86b377SEli Cohen 22721a86b377SEli Cohen if (change_map) 2273ae0428deSEli Cohen return mlx5_vdpa_change_map(mvdev, iotlb); 22741a86b377SEli Cohen 22751a86b377SEli Cohen return 0; 22761a86b377SEli Cohen } 22771a86b377SEli Cohen 22781a86b377SEli Cohen static void mlx5_vdpa_free(struct vdpa_device *vdev) 22791a86b377SEli Cohen { 22801a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22817c9f131fSEli Cohen struct mlx5_core_dev *pfmdev; 22821a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 22831a86b377SEli Cohen 22841a86b377SEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 22851a86b377SEli Cohen 22861a86b377SEli Cohen free_resources(ndev); 22876f5312f8SEli Cohen mlx5_vdpa_destroy_mr(mvdev); 22887c9f131fSEli Cohen if (!is_zero_ether_addr(ndev->config.mac)) { 22897c9f131fSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 22907c9f131fSEli Cohen mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); 22917c9f131fSEli Cohen } 22921a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 22931a86b377SEli Cohen mutex_destroy(&ndev->reslock); 22941a86b377SEli Cohen } 22951a86b377SEli Cohen 22961a86b377SEli Cohen static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx) 22971a86b377SEli Cohen { 2298b57c46cbSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 22991a86b377SEli Cohen struct vdpa_notification_area ret = {}; 2300b57c46cbSEli Cohen struct mlx5_vdpa_net *ndev; 2301b57c46cbSEli Cohen phys_addr_t addr; 23021a86b377SEli Cohen 23035262912eSEli Cohen if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2304e4fc6650SEli Cohen return ret; 2305e4fc6650SEli Cohen 2306b57c46cbSEli Cohen /* If SF BAR size is smaller than PAGE_SIZE, do not use direct 2307b57c46cbSEli Cohen * notification to avoid the risk of mapping pages that contain BAR of more 2308b57c46cbSEli Cohen * than one SF 2309b57c46cbSEli Cohen */ 2310b57c46cbSEli Cohen if (MLX5_CAP_GEN(mvdev->mdev, log_min_sf_size) + 12 < PAGE_SHIFT) 2311b57c46cbSEli Cohen return ret; 2312b57c46cbSEli Cohen 2313b57c46cbSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 2314b57c46cbSEli Cohen addr = (phys_addr_t)ndev->mvdev.res.phys_kick_addr; 2315b57c46cbSEli Cohen ret.addr = addr; 2316b57c46cbSEli Cohen ret.size = PAGE_SIZE; 23171a86b377SEli Cohen return ret; 23181a86b377SEli Cohen } 23191a86b377SEli Cohen 23201a86b377SEli Cohen static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) 23211a86b377SEli Cohen { 23221a86b377SEli Cohen return -EOPNOTSUPP; 23231a86b377SEli Cohen } 23241a86b377SEli Cohen 2325a64917bcSEli Cohen static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) 2326a64917bcSEli Cohen { 2327a64917bcSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2328a64917bcSEli Cohen 2329a64917bcSEli Cohen return mvdev->actual_features; 2330a64917bcSEli Cohen } 2331a64917bcSEli Cohen 23321a86b377SEli Cohen static const struct vdpa_config_ops mlx5_vdpa_ops = { 23331a86b377SEli Cohen .set_vq_address = mlx5_vdpa_set_vq_address, 23341a86b377SEli Cohen .set_vq_num = mlx5_vdpa_set_vq_num, 23351a86b377SEli Cohen .kick_vq = mlx5_vdpa_kick_vq, 23361a86b377SEli Cohen .set_vq_cb = mlx5_vdpa_set_vq_cb, 23371a86b377SEli Cohen .set_vq_ready = mlx5_vdpa_set_vq_ready, 23381a86b377SEli Cohen .get_vq_ready = mlx5_vdpa_get_vq_ready, 23391a86b377SEli Cohen .set_vq_state = mlx5_vdpa_set_vq_state, 23401a86b377SEli Cohen .get_vq_state = mlx5_vdpa_get_vq_state, 23411a86b377SEli Cohen .get_vq_notification = mlx5_get_vq_notification, 23421a86b377SEli Cohen .get_vq_irq = mlx5_get_vq_irq, 23431a86b377SEli Cohen .get_vq_align = mlx5_vdpa_get_vq_align, 2344a64917bcSEli Cohen .get_device_features = mlx5_vdpa_get_device_features, 2345a64917bcSEli Cohen .set_driver_features = mlx5_vdpa_set_driver_features, 2346a64917bcSEli Cohen .get_driver_features = mlx5_vdpa_get_driver_features, 23471a86b377SEli Cohen .set_config_cb = mlx5_vdpa_set_config_cb, 23481a86b377SEli Cohen .get_vq_num_max = mlx5_vdpa_get_vq_num_max, 23491a86b377SEli Cohen .get_device_id = mlx5_vdpa_get_device_id, 23501a86b377SEli Cohen .get_vendor_id = mlx5_vdpa_get_vendor_id, 23511a86b377SEli Cohen .get_status = mlx5_vdpa_get_status, 23521a86b377SEli Cohen .set_status = mlx5_vdpa_set_status, 23530686082dSXie Yongji .reset = mlx5_vdpa_reset, 2354442706f9SStefano Garzarella .get_config_size = mlx5_vdpa_get_config_size, 23551a86b377SEli Cohen .get_config = mlx5_vdpa_get_config, 23561a86b377SEli Cohen .set_config = mlx5_vdpa_set_config, 23571a86b377SEli Cohen .get_generation = mlx5_vdpa_get_generation, 23581a86b377SEli Cohen .set_map = mlx5_vdpa_set_map, 23591a86b377SEli Cohen .free = mlx5_vdpa_free, 23601a86b377SEli Cohen }; 23611a86b377SEli Cohen 2362d084d996SSi-Wei Liu static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) 2363d084d996SSi-Wei Liu { 2364d084d996SSi-Wei Liu u16 hw_mtu; 2365d084d996SSi-Wei Liu int err; 2366d084d996SSi-Wei Liu 2367d084d996SSi-Wei Liu err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2368d084d996SSi-Wei Liu if (err) 2369d084d996SSi-Wei Liu return err; 2370d084d996SSi-Wei Liu 2371d084d996SSi-Wei Liu *mtu = hw_mtu - MLX5V_ETH_HARD_MTU; 2372d084d996SSi-Wei Liu return 0; 2373d084d996SSi-Wei Liu } 2374d084d996SSi-Wei Liu 23751a86b377SEli Cohen static int alloc_resources(struct mlx5_vdpa_net *ndev) 23761a86b377SEli Cohen { 23771a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 23781a86b377SEli Cohen int err; 23791a86b377SEli Cohen 23801a86b377SEli Cohen if (res->valid) { 23811a86b377SEli Cohen mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); 23821a86b377SEli Cohen return -EEXIST; 23831a86b377SEli Cohen } 23841a86b377SEli Cohen 23851a86b377SEli Cohen err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); 23861a86b377SEli Cohen if (err) 23871a86b377SEli Cohen return err; 23881a86b377SEli Cohen 23891a86b377SEli Cohen err = create_tis(ndev); 23901a86b377SEli Cohen if (err) 23911a86b377SEli Cohen goto err_tis; 23921a86b377SEli Cohen 23931a86b377SEli Cohen res->valid = true; 23941a86b377SEli Cohen 23951a86b377SEli Cohen return 0; 23961a86b377SEli Cohen 23971a86b377SEli Cohen err_tis: 23981a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 23991a86b377SEli Cohen return err; 24001a86b377SEli Cohen } 24011a86b377SEli Cohen 24021a86b377SEli Cohen static void free_resources(struct mlx5_vdpa_net *ndev) 24031a86b377SEli Cohen { 24041a86b377SEli Cohen struct mlx5_vdpa_net_resources *res = &ndev->res; 24051a86b377SEli Cohen 24061a86b377SEli Cohen if (!res->valid) 24071a86b377SEli Cohen return; 24081a86b377SEli Cohen 24091a86b377SEli Cohen destroy_tis(ndev); 24101a86b377SEli Cohen mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 24111a86b377SEli Cohen res->valid = false; 24121a86b377SEli Cohen } 24131a86b377SEli Cohen 24141a86b377SEli Cohen static void init_mvqs(struct mlx5_vdpa_net *ndev) 24151a86b377SEli Cohen { 24161a86b377SEli Cohen struct mlx5_vdpa_virtqueue *mvq; 24171a86b377SEli Cohen int i; 24181a86b377SEli Cohen 24191a86b377SEli Cohen for (i = 0; i < 2 * mlx5_vdpa_max_qps(ndev->mvdev.max_vqs); ++i) { 24201a86b377SEli Cohen mvq = &ndev->vqs[i]; 24211a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 24221a86b377SEli Cohen mvq->index = i; 24231a86b377SEli Cohen mvq->ndev = ndev; 24241a86b377SEli Cohen mvq->fwqp.fw = true; 24251a86b377SEli Cohen } 24261a86b377SEli Cohen for (; i < ndev->mvdev.max_vqs; i++) { 24271a86b377SEli Cohen mvq = &ndev->vqs[i]; 24281a86b377SEli Cohen memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 24291a86b377SEli Cohen mvq->index = i; 24301a86b377SEli Cohen mvq->ndev = ndev; 24311a86b377SEli Cohen } 24321a86b377SEli Cohen } 24331a86b377SEli Cohen 243458926c8aSEli Cohen struct mlx5_vdpa_mgmtdev { 243558926c8aSEli Cohen struct vdpa_mgmt_dev mgtdev; 243658926c8aSEli Cohen struct mlx5_adev *madev; 243758926c8aSEli Cohen struct mlx5_vdpa_net *ndev; 243858926c8aSEli Cohen }; 243958926c8aSEli Cohen 2440edf747afSEli Cohen static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) 2441edf747afSEli Cohen { 2442edf747afSEli Cohen u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; 2443edf747afSEli Cohen u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; 2444edf747afSEli Cohen int err; 2445edf747afSEli Cohen 2446edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE); 2447edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, op_mod, opmod); 2448edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, vport_number, vport); 2449edf747afSEli Cohen if (vport) 2450edf747afSEli Cohen MLX5_SET(query_vport_state_in, in, other_vport, 1); 2451edf747afSEli Cohen 2452edf747afSEli Cohen err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out); 2453edf747afSEli Cohen if (err) 2454edf747afSEli Cohen return 0; 2455edf747afSEli Cohen 2456edf747afSEli Cohen return MLX5_GET(query_vport_state_out, out, state); 2457edf747afSEli Cohen } 2458edf747afSEli Cohen 2459edf747afSEli Cohen static bool get_link_state(struct mlx5_vdpa_dev *mvdev) 2460edf747afSEli Cohen { 2461edf747afSEli Cohen if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) == 2462edf747afSEli Cohen VPORT_STATE_UP) 2463edf747afSEli Cohen return true; 2464edf747afSEli Cohen 2465edf747afSEli Cohen return false; 2466edf747afSEli Cohen } 2467edf747afSEli Cohen 2468edf747afSEli Cohen static void update_carrier(struct work_struct *work) 2469edf747afSEli Cohen { 2470edf747afSEli Cohen struct mlx5_vdpa_wq_ent *wqent; 2471edf747afSEli Cohen struct mlx5_vdpa_dev *mvdev; 2472edf747afSEli Cohen struct mlx5_vdpa_net *ndev; 2473edf747afSEli Cohen 2474edf747afSEli Cohen wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 2475edf747afSEli Cohen mvdev = wqent->mvdev; 2476edf747afSEli Cohen ndev = to_mlx5_vdpa_ndev(mvdev); 2477edf747afSEli Cohen if (get_link_state(mvdev)) 2478edf747afSEli Cohen ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 2479edf747afSEli Cohen else 2480edf747afSEli Cohen ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 2481edf747afSEli Cohen 2482edf747afSEli Cohen if (ndev->config_cb.callback) 2483edf747afSEli Cohen ndev->config_cb.callback(ndev->config_cb.private); 2484edf747afSEli Cohen 2485edf747afSEli Cohen kfree(wqent); 2486edf747afSEli Cohen } 2487edf747afSEli Cohen 2488edf747afSEli Cohen static int event_handler(struct notifier_block *nb, unsigned long event, void *param) 2489edf747afSEli Cohen { 2490edf747afSEli Cohen struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb); 2491edf747afSEli Cohen struct mlx5_eqe *eqe = param; 2492edf747afSEli Cohen int ret = NOTIFY_DONE; 2493edf747afSEli Cohen struct mlx5_vdpa_wq_ent *wqent; 2494edf747afSEli Cohen 2495edf747afSEli Cohen if (event == MLX5_EVENT_TYPE_PORT_CHANGE) { 2496edf747afSEli Cohen switch (eqe->sub_type) { 2497edf747afSEli Cohen case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 2498edf747afSEli Cohen case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 2499edf747afSEli Cohen wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); 2500edf747afSEli Cohen if (!wqent) 2501edf747afSEli Cohen return NOTIFY_DONE; 2502edf747afSEli Cohen 2503edf747afSEli Cohen wqent->mvdev = &ndev->mvdev; 2504edf747afSEli Cohen INIT_WORK(&wqent->work, update_carrier); 2505edf747afSEli Cohen queue_work(ndev->mvdev.wq, &wqent->work); 2506edf747afSEli Cohen ret = NOTIFY_OK; 2507edf747afSEli Cohen break; 2508edf747afSEli Cohen default: 2509edf747afSEli Cohen return NOTIFY_DONE; 2510edf747afSEli Cohen } 2511edf747afSEli Cohen return ret; 2512edf747afSEli Cohen } 2513edf747afSEli Cohen return ret; 2514edf747afSEli Cohen } 2515edf747afSEli Cohen 2516d8ca2fa5SParav Pandit static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, 2517d8ca2fa5SParav Pandit const struct vdpa_dev_set_config *add_config) 25181a86b377SEli Cohen { 251958926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 25201a86b377SEli Cohen struct virtio_net_config *config; 25217c9f131fSEli Cohen struct mlx5_core_dev *pfmdev; 25221a86b377SEli Cohen struct mlx5_vdpa_dev *mvdev; 25231a86b377SEli Cohen struct mlx5_vdpa_net *ndev; 252458926c8aSEli Cohen struct mlx5_core_dev *mdev; 25251a86b377SEli Cohen u32 max_vqs; 2526246fd1caSEli Cohen u16 mtu; 25271a86b377SEli Cohen int err; 25281a86b377SEli Cohen 252958926c8aSEli Cohen if (mgtdev->ndev) 253058926c8aSEli Cohen return -ENOSPC; 253158926c8aSEli Cohen 253258926c8aSEli Cohen mdev = mgtdev->madev->mdev; 2533879753c8SEli Cohen if (!(MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_queue_type) & 2534879753c8SEli Cohen MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)) { 2535879753c8SEli Cohen dev_warn(mdev->device, "missing support for split virtqueues\n"); 2536879753c8SEli Cohen return -EOPNOTSUPP; 2537879753c8SEli Cohen } 2538879753c8SEli Cohen 25391a86b377SEli Cohen /* we save one virtqueue for control virtqueue should we require it */ 25401a86b377SEli Cohen max_vqs = MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues); 25411a86b377SEli Cohen max_vqs = min_t(u32, max_vqs, MLX5_MAX_SUPPORTED_VQS); 25421a86b377SEli Cohen 25431a86b377SEli Cohen ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, 2544d8945ec4SXie Yongji name, false); 25451a86b377SEli Cohen if (IS_ERR(ndev)) 254674c9729dSLeon Romanovsky return PTR_ERR(ndev); 25471a86b377SEli Cohen 25481a86b377SEli Cohen ndev->mvdev.max_vqs = max_vqs; 25491a86b377SEli Cohen mvdev = &ndev->mvdev; 25501a86b377SEli Cohen mvdev->mdev = mdev; 25511a86b377SEli Cohen init_mvqs(ndev); 25521a86b377SEli Cohen mutex_init(&ndev->reslock); 25531a86b377SEli Cohen config = &ndev->config; 2554246fd1caSEli Cohen err = query_mtu(mdev, &mtu); 25551a86b377SEli Cohen if (err) 25561a86b377SEli Cohen goto err_mtu; 25571a86b377SEli Cohen 2558246fd1caSEli Cohen ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, mtu); 25591a86b377SEli Cohen 2560edf747afSEli Cohen if (get_link_state(mvdev)) 2561edf747afSEli Cohen ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 2562edf747afSEli Cohen else 2563edf747afSEli Cohen ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 2564edf747afSEli Cohen 2565a007d940SEli Cohen if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) { 2566a007d940SEli Cohen memcpy(ndev->config.mac, add_config->net.mac, ETH_ALEN); 2567a007d940SEli Cohen } else { 25681a86b377SEli Cohen err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac); 25691a86b377SEli Cohen if (err) 25701a86b377SEli Cohen goto err_mtu; 2571a007d940SEli Cohen } 25721a86b377SEli Cohen 25737c9f131fSEli Cohen if (!is_zero_ether_addr(config->mac)) { 25747c9f131fSEli Cohen pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev)); 25757c9f131fSEli Cohen err = mlx5_mpfs_add_mac(pfmdev, config->mac); 25767c9f131fSEli Cohen if (err) 25777c9f131fSEli Cohen goto err_mtu; 25785262912eSEli Cohen 25795262912eSEli Cohen ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MAC); 25807c9f131fSEli Cohen } 25817c9f131fSEli Cohen 25825262912eSEli Cohen config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, mlx5_vdpa_max_qps(max_vqs)); 25837d23dcdfSEli Cohen mvdev->vdev.dma_dev = &mdev->pdev->dev; 25841a86b377SEli Cohen err = mlx5_vdpa_alloc_resources(&ndev->mvdev); 25851a86b377SEli Cohen if (err) 25867c9f131fSEli Cohen goto err_mpfs; 25871a86b377SEli Cohen 25886f5312f8SEli Cohen if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 25896f5312f8SEli Cohen err = mlx5_vdpa_create_mr(mvdev, NULL); 25901a86b377SEli Cohen if (err) 25911a86b377SEli Cohen goto err_res; 25926f5312f8SEli Cohen } 25936f5312f8SEli Cohen 25946f5312f8SEli Cohen err = alloc_resources(ndev); 25956f5312f8SEli Cohen if (err) 25966f5312f8SEli Cohen goto err_mr; 25971a86b377SEli Cohen 2598218bdd20SEli Cohen mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_wq"); 25995262912eSEli Cohen if (!mvdev->wq) { 26005262912eSEli Cohen err = -ENOMEM; 26015262912eSEli Cohen goto err_res2; 26025262912eSEli Cohen } 26035262912eSEli Cohen 2604edf747afSEli Cohen ndev->nb.notifier_call = event_handler; 2605edf747afSEli Cohen mlx5_notifier_register(mdev, &ndev->nb); 26065262912eSEli Cohen ndev->cur_num_vqs = 2 * mlx5_vdpa_max_qps(max_vqs); 260758926c8aSEli Cohen mvdev->vdev.mdev = &mgtdev->mgtdev; 26085262912eSEli Cohen err = _vdpa_register_device(&mvdev->vdev, ndev->cur_num_vqs + 1); 26091a86b377SEli Cohen if (err) 26101a86b377SEli Cohen goto err_reg; 26111a86b377SEli Cohen 261258926c8aSEli Cohen mgtdev->ndev = ndev; 261374c9729dSLeon Romanovsky return 0; 26141a86b377SEli Cohen 26151a86b377SEli Cohen err_reg: 26165262912eSEli Cohen destroy_workqueue(mvdev->wq); 26175262912eSEli Cohen err_res2: 26181a86b377SEli Cohen free_resources(ndev); 26196f5312f8SEli Cohen err_mr: 26206f5312f8SEli Cohen mlx5_vdpa_destroy_mr(mvdev); 26211a86b377SEli Cohen err_res: 26221a86b377SEli Cohen mlx5_vdpa_free_resources(&ndev->mvdev); 26237c9f131fSEli Cohen err_mpfs: 26247c9f131fSEli Cohen if (!is_zero_ether_addr(config->mac)) 26257c9f131fSEli Cohen mlx5_mpfs_del_mac(pfmdev, config->mac); 26261a86b377SEli Cohen err_mtu: 26271a86b377SEli Cohen mutex_destroy(&ndev->reslock); 26281a86b377SEli Cohen put_device(&mvdev->vdev.dev); 262974c9729dSLeon Romanovsky return err; 26301a86b377SEli Cohen } 26311a86b377SEli Cohen 263258926c8aSEli Cohen static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev) 263358926c8aSEli Cohen { 263458926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 26355262912eSEli Cohen struct mlx5_vdpa_dev *mvdev = to_mvdev(dev); 2636edf747afSEli Cohen struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 263758926c8aSEli Cohen 2638edf747afSEli Cohen mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 26395262912eSEli Cohen destroy_workqueue(mvdev->wq); 264058926c8aSEli Cohen _vdpa_unregister_device(dev); 264158926c8aSEli Cohen mgtdev->ndev = NULL; 264258926c8aSEli Cohen } 264358926c8aSEli Cohen 264458926c8aSEli Cohen static const struct vdpa_mgmtdev_ops mdev_ops = { 264558926c8aSEli Cohen .dev_add = mlx5_vdpa_dev_add, 264658926c8aSEli Cohen .dev_del = mlx5_vdpa_dev_del, 264758926c8aSEli Cohen }; 264858926c8aSEli Cohen 264958926c8aSEli Cohen static struct virtio_device_id id_table[] = { 265058926c8aSEli Cohen { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 265158926c8aSEli Cohen { 0 }, 265258926c8aSEli Cohen }; 265358926c8aSEli Cohen 265458926c8aSEli Cohen static int mlx5v_probe(struct auxiliary_device *adev, 265558926c8aSEli Cohen const struct auxiliary_device_id *id) 265658926c8aSEli Cohen 265758926c8aSEli Cohen { 265858926c8aSEli Cohen struct mlx5_adev *madev = container_of(adev, struct mlx5_adev, adev); 265958926c8aSEli Cohen struct mlx5_core_dev *mdev = madev->mdev; 266058926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev; 266158926c8aSEli Cohen int err; 266258926c8aSEli Cohen 266358926c8aSEli Cohen mgtdev = kzalloc(sizeof(*mgtdev), GFP_KERNEL); 266458926c8aSEli Cohen if (!mgtdev) 266558926c8aSEli Cohen return -ENOMEM; 266658926c8aSEli Cohen 266758926c8aSEli Cohen mgtdev->mgtdev.ops = &mdev_ops; 266858926c8aSEli Cohen mgtdev->mgtdev.device = mdev->device; 266958926c8aSEli Cohen mgtdev->mgtdev.id_table = id_table; 2670a007d940SEli Cohen mgtdev->mgtdev.config_attr_mask = (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR); 267158926c8aSEli Cohen mgtdev->madev = madev; 267258926c8aSEli Cohen 267358926c8aSEli Cohen err = vdpa_mgmtdev_register(&mgtdev->mgtdev); 267458926c8aSEli Cohen if (err) 267558926c8aSEli Cohen goto reg_err; 267658926c8aSEli Cohen 267758926c8aSEli Cohen dev_set_drvdata(&adev->dev, mgtdev); 267858926c8aSEli Cohen 267958926c8aSEli Cohen return 0; 268058926c8aSEli Cohen 268158926c8aSEli Cohen reg_err: 268258926c8aSEli Cohen kfree(mgtdev); 268358926c8aSEli Cohen return err; 268458926c8aSEli Cohen } 268558926c8aSEli Cohen 268674c9729dSLeon Romanovsky static void mlx5v_remove(struct auxiliary_device *adev) 26871a86b377SEli Cohen { 268858926c8aSEli Cohen struct mlx5_vdpa_mgmtdev *mgtdev; 268974c9729dSLeon Romanovsky 269058926c8aSEli Cohen mgtdev = dev_get_drvdata(&adev->dev); 269158926c8aSEli Cohen vdpa_mgmtdev_unregister(&mgtdev->mgtdev); 269258926c8aSEli Cohen kfree(mgtdev); 26931a86b377SEli Cohen } 269474c9729dSLeon Romanovsky 269574c9729dSLeon Romanovsky static const struct auxiliary_device_id mlx5v_id_table[] = { 269674c9729dSLeon Romanovsky { .name = MLX5_ADEV_NAME ".vnet", }, 269774c9729dSLeon Romanovsky {}, 269874c9729dSLeon Romanovsky }; 269974c9729dSLeon Romanovsky 270074c9729dSLeon Romanovsky MODULE_DEVICE_TABLE(auxiliary, mlx5v_id_table); 270174c9729dSLeon Romanovsky 270274c9729dSLeon Romanovsky static struct auxiliary_driver mlx5v_driver = { 270374c9729dSLeon Romanovsky .name = "vnet", 270474c9729dSLeon Romanovsky .probe = mlx5v_probe, 270574c9729dSLeon Romanovsky .remove = mlx5v_remove, 270674c9729dSLeon Romanovsky .id_table = mlx5v_id_table, 270774c9729dSLeon Romanovsky }; 270874c9729dSLeon Romanovsky 270974c9729dSLeon Romanovsky module_auxiliary_driver(mlx5v_driver); 2710