1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 3 4 #include <linux/module.h> 5 #include <linux/vdpa.h> 6 #include <linux/vringh.h> 7 #include <uapi/linux/virtio_net.h> 8 #include <uapi/linux/virtio_ids.h> 9 #include <uapi/linux/vdpa.h> 10 #include <linux/virtio_config.h> 11 #include <linux/auxiliary_bus.h> 12 #include <linux/mlx5/cq.h> 13 #include <linux/mlx5/qp.h> 14 #include <linux/mlx5/device.h> 15 #include <linux/mlx5/driver.h> 16 #include <linux/mlx5/vport.h> 17 #include <linux/mlx5/fs.h> 18 #include <linux/mlx5/mlx5_ifc_vdpa.h> 19 #include <linux/mlx5/mpfs.h> 20 #include "mlx5_vdpa.h" 21 #include "mlx5_vnet.h" 22 23 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 24 MODULE_DESCRIPTION("Mellanox VDPA driver"); 25 MODULE_LICENSE("Dual BSD/GPL"); 26 27 #define VALID_FEATURES_MASK \ 28 (BIT_ULL(VIRTIO_NET_F_CSUM) | BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | \ 29 BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_MAC) | \ 30 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | \ 31 BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | \ 32 BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | BIT_ULL(VIRTIO_NET_F_HOST_ECN) | BIT_ULL(VIRTIO_NET_F_HOST_UFO) | \ 33 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | BIT_ULL(VIRTIO_NET_F_STATUS) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | \ 34 BIT_ULL(VIRTIO_NET_F_CTRL_RX) | BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | \ 35 BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE) | \ 36 BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | \ 37 BIT_ULL(VIRTIO_NET_F_RSS) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY) | \ 38 BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX) | BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY) | \ 39 BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | \ 40 BIT_ULL(VIRTIO_F_RING_PACKED) | BIT_ULL(VIRTIO_F_ORDER_PLATFORM) | BIT_ULL(VIRTIO_F_SR_IOV)) 41 42 #define VALID_STATUS_MASK \ 43 (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \ 44 VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED) 45 46 #define MLX5_FEATURE(_mvdev, _feature) (!!((_mvdev)->actual_features & BIT_ULL(_feature))) 47 48 #define MLX5V_UNTAGGED 0x1000 49 50 struct mlx5_vdpa_cq_buf { 51 struct mlx5_frag_buf_ctrl fbc; 52 struct mlx5_frag_buf frag_buf; 53 int cqe_size; 54 int nent; 55 }; 56 57 struct mlx5_vdpa_cq { 58 struct mlx5_core_cq mcq; 59 struct mlx5_vdpa_cq_buf buf; 60 struct mlx5_db db; 61 int cqe; 62 }; 63 64 struct mlx5_vdpa_umem { 65 struct mlx5_frag_buf_ctrl fbc; 66 struct mlx5_frag_buf frag_buf; 67 int size; 68 u32 id; 69 }; 70 71 struct mlx5_vdpa_qp { 72 struct mlx5_core_qp mqp; 73 struct mlx5_frag_buf frag_buf; 74 struct mlx5_db db; 75 u16 head; 76 bool fw; 77 }; 78 79 struct mlx5_vq_restore_info { 80 u32 num_ent; 81 u64 desc_addr; 82 u64 device_addr; 83 u64 driver_addr; 84 u16 avail_index; 85 u16 used_index; 86 bool ready; 87 bool restore; 88 }; 89 90 struct mlx5_vdpa_virtqueue { 91 bool ready; 92 u64 desc_addr; 93 u64 device_addr; 94 u64 driver_addr; 95 u32 num_ent; 96 97 /* Resources for implementing the notification channel from the device 98 * to the driver. fwqp is the firmware end of an RC connection; the 99 * other end is vqqp used by the driver. cq is where completions are 100 * reported. 101 */ 102 struct mlx5_vdpa_cq cq; 103 struct mlx5_vdpa_qp fwqp; 104 struct mlx5_vdpa_qp vqqp; 105 106 /* umem resources are required for the virtqueue operation. They're use 107 * is internal and they must be provided by the driver. 108 */ 109 struct mlx5_vdpa_umem umem1; 110 struct mlx5_vdpa_umem umem2; 111 struct mlx5_vdpa_umem umem3; 112 113 u32 counter_set_id; 114 bool initialized; 115 int index; 116 u32 virtq_id; 117 struct mlx5_vdpa_net *ndev; 118 u16 avail_idx; 119 u16 used_idx; 120 int fw_state; 121 122 /* keep last in the struct */ 123 struct mlx5_vq_restore_info ri; 124 }; 125 126 static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx) 127 { 128 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) { 129 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 130 return idx < 2; 131 else 132 return idx < 3; 133 } 134 135 return idx <= mvdev->max_idx; 136 } 137 138 static void free_resources(struct mlx5_vdpa_net *ndev); 139 static void init_mvqs(struct mlx5_vdpa_net *ndev); 140 static int setup_driver(struct mlx5_vdpa_dev *mvdev); 141 static void teardown_driver(struct mlx5_vdpa_net *ndev); 142 143 static bool mlx5_vdpa_debug; 144 145 #define MLX5_CVQ_MAX_ENT 16 146 147 #define MLX5_LOG_VIO_FLAG(_feature) \ 148 do { \ 149 if (features & BIT_ULL(_feature)) \ 150 mlx5_vdpa_info(mvdev, "%s\n", #_feature); \ 151 } while (0) 152 153 #define MLX5_LOG_VIO_STAT(_status) \ 154 do { \ 155 if (status & (_status)) \ 156 mlx5_vdpa_info(mvdev, "%s\n", #_status); \ 157 } while (0) 158 159 /* TODO: cross-endian support */ 160 static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev) 161 { 162 return virtio_legacy_is_little_endian() || 163 (mvdev->actual_features & BIT_ULL(VIRTIO_F_VERSION_1)); 164 } 165 166 static u16 mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev *mvdev, __virtio16 val) 167 { 168 return __virtio16_to_cpu(mlx5_vdpa_is_little_endian(mvdev), val); 169 } 170 171 static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val) 172 { 173 return __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), val); 174 } 175 176 static u16 ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev) 177 { 178 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) 179 return 2; 180 181 return mvdev->max_vqs; 182 } 183 184 static bool is_ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev, u16 idx) 185 { 186 return idx == ctrl_vq_idx(mvdev); 187 } 188 189 static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set) 190 { 191 if (status & ~VALID_STATUS_MASK) 192 mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n", 193 status & ~VALID_STATUS_MASK); 194 195 if (!mlx5_vdpa_debug) 196 return; 197 198 mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get"); 199 if (set && !status) { 200 mlx5_vdpa_info(mvdev, "driver resets the device\n"); 201 return; 202 } 203 204 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE); 205 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER); 206 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK); 207 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK); 208 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET); 209 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED); 210 } 211 212 static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set) 213 { 214 if (features & ~VALID_FEATURES_MASK) 215 mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n", 216 features & ~VALID_FEATURES_MASK); 217 218 if (!mlx5_vdpa_debug) 219 return; 220 221 mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads"); 222 if (!features) 223 mlx5_vdpa_info(mvdev, "all feature bits are cleared\n"); 224 225 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM); 226 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM); 227 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 228 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU); 229 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC); 230 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4); 231 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6); 232 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN); 233 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO); 234 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4); 235 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6); 236 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN); 237 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO); 238 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF); 239 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS); 240 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ); 241 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX); 242 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN); 243 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA); 244 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE); 245 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ); 246 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR); 247 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT); 248 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS); 249 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT); 250 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY); 251 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX); 252 MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY); 253 MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT); 254 MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1); 255 MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM); 256 MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED); 257 MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM); 258 MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV); 259 } 260 261 static int create_tis(struct mlx5_vdpa_net *ndev) 262 { 263 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; 264 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 265 void *tisc; 266 int err; 267 268 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 269 MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); 270 err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); 271 if (err) 272 mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err); 273 274 return err; 275 } 276 277 static void destroy_tis(struct mlx5_vdpa_net *ndev) 278 { 279 mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); 280 } 281 282 #define MLX5_VDPA_CQE_SIZE 64 283 #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE) 284 285 static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) 286 { 287 struct mlx5_frag_buf *frag_buf = &buf->frag_buf; 288 u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE; 289 u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE; 290 int err; 291 292 err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, 293 ndev->mvdev.mdev->priv.numa_node); 294 if (err) 295 return err; 296 297 mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc); 298 299 buf->cqe_size = MLX5_VDPA_CQE_SIZE; 300 buf->nent = nent; 301 302 return 0; 303 } 304 305 static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) 306 { 307 struct mlx5_frag_buf *frag_buf = &umem->frag_buf; 308 309 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, 310 ndev->mvdev.mdev->priv.numa_node); 311 } 312 313 static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) 314 { 315 mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); 316 } 317 318 static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n) 319 { 320 return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n); 321 } 322 323 static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf) 324 { 325 struct mlx5_cqe64 *cqe64; 326 void *cqe; 327 int i; 328 329 for (i = 0; i < buf->nent; i++) { 330 cqe = get_cqe(vcq, i); 331 cqe64 = cqe; 332 cqe64->op_own = MLX5_CQE_INVALID << 4; 333 } 334 } 335 336 static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n) 337 { 338 struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1)); 339 340 if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && 341 !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe))) 342 return cqe64; 343 344 return NULL; 345 } 346 347 static void rx_post(struct mlx5_vdpa_qp *vqp, int n) 348 { 349 vqp->head += n; 350 vqp->db.db[0] = cpu_to_be32(vqp->head); 351 } 352 353 static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, 354 struct mlx5_vdpa_virtqueue *mvq, u32 num_ent) 355 { 356 struct mlx5_vdpa_qp *vqp; 357 __be64 *pas; 358 void *qpc; 359 360 vqp = fw ? &mvq->fwqp : &mvq->vqqp; 361 MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); 362 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 363 if (vqp->fw) { 364 /* Firmware QP is allocated by the driver for the firmware's 365 * use so we can skip part of the params as they will be chosen by firmware 366 */ 367 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 368 MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ); 369 MLX5_SET(qpc, qpc, no_sq, 1); 370 return; 371 } 372 373 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 374 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 375 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 376 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 377 MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); 378 MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 379 MLX5_SET(qpc, qpc, no_sq, 1); 380 MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn); 381 MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent)); 382 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 383 pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas); 384 mlx5_fill_page_frag_array(&vqp->frag_buf, pas); 385 } 386 387 static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) 388 { 389 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, 390 num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf, 391 ndev->mvdev.mdev->priv.numa_node); 392 } 393 394 static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 395 { 396 mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); 397 } 398 399 static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 400 struct mlx5_vdpa_qp *vqp) 401 { 402 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 403 int inlen = MLX5_ST_SZ_BYTES(create_qp_in); 404 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 405 void *qpc; 406 void *in; 407 int err; 408 409 if (!vqp->fw) { 410 vqp = &mvq->vqqp; 411 err = rq_buf_alloc(ndev, vqp, mvq->num_ent); 412 if (err) 413 return err; 414 415 err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); 416 if (err) 417 goto err_db; 418 inlen += vqp->frag_buf.npages * sizeof(__be64); 419 } 420 421 in = kzalloc(inlen, GFP_KERNEL); 422 if (!in) { 423 err = -ENOMEM; 424 goto err_kzalloc; 425 } 426 427 qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); 428 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 429 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 430 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 431 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 432 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 433 if (!vqp->fw) 434 MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma); 435 MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 436 err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 437 kfree(in); 438 if (err) 439 goto err_kzalloc; 440 441 vqp->mqp.uid = ndev->mvdev.res.uid; 442 vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn); 443 444 if (!vqp->fw) 445 rx_post(vqp, mvq->num_ent); 446 447 return 0; 448 449 err_kzalloc: 450 if (!vqp->fw) 451 mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 452 err_db: 453 if (!vqp->fw) 454 rq_buf_free(ndev, vqp); 455 456 return err; 457 } 458 459 static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 460 { 461 u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 462 463 MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 464 MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn); 465 MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); 466 if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) 467 mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); 468 if (!vqp->fw) { 469 mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 470 rq_buf_free(ndev, vqp); 471 } 472 } 473 474 static void *next_cqe_sw(struct mlx5_vdpa_cq *cq) 475 { 476 return get_sw_cqe(cq, cq->mcq.cons_index); 477 } 478 479 static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq) 480 { 481 struct mlx5_cqe64 *cqe64; 482 483 cqe64 = next_cqe_sw(vcq); 484 if (!cqe64) 485 return -EAGAIN; 486 487 vcq->mcq.cons_index++; 488 return 0; 489 } 490 491 static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num) 492 { 493 struct mlx5_vdpa_net *ndev = mvq->ndev; 494 struct vdpa_callback *event_cb; 495 496 event_cb = &ndev->event_cbs[mvq->index]; 497 mlx5_cq_set_ci(&mvq->cq.mcq); 498 499 /* make sure CQ cosumer update is visible to the hardware before updating 500 * RX doorbell record. 501 */ 502 dma_wmb(); 503 rx_post(&mvq->vqqp, num); 504 if (event_cb->callback) 505 event_cb->callback(event_cb->private); 506 } 507 508 static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) 509 { 510 struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq); 511 struct mlx5_vdpa_net *ndev = mvq->ndev; 512 void __iomem *uar_page = ndev->mvdev.res.uar->map; 513 int num = 0; 514 515 while (!mlx5_vdpa_poll_one(&mvq->cq)) { 516 num++; 517 if (num > mvq->num_ent / 2) { 518 /* If completions keep coming while we poll, we want to 519 * let the hardware know that we consumed them by 520 * updating the doorbell record. We also let vdpa core 521 * know about this so it passes it on the virtio driver 522 * on the guest. 523 */ 524 mlx5_vdpa_handle_completions(mvq, num); 525 num = 0; 526 } 527 } 528 529 if (num) 530 mlx5_vdpa_handle_completions(mvq, num); 531 532 mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 533 } 534 535 static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) 536 { 537 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 538 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 539 void __iomem *uar_page = ndev->mvdev.res.uar->map; 540 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 541 struct mlx5_vdpa_cq *vcq = &mvq->cq; 542 __be64 *pas; 543 int inlen; 544 void *cqc; 545 void *in; 546 int err; 547 int eqn; 548 549 err = mlx5_db_alloc(mdev, &vcq->db); 550 if (err) 551 return err; 552 553 vcq->mcq.set_ci_db = vcq->db.db; 554 vcq->mcq.arm_db = vcq->db.db + 1; 555 vcq->mcq.cqe_sz = 64; 556 557 err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); 558 if (err) 559 goto err_db; 560 561 cq_frag_buf_init(vcq, &vcq->buf); 562 563 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 564 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages; 565 in = kzalloc(inlen, GFP_KERNEL); 566 if (!in) { 567 err = -ENOMEM; 568 goto err_vzalloc; 569 } 570 571 MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); 572 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 573 mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas); 574 575 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 576 MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 577 578 /* Use vector 0 by default. Consider adding code to choose least used 579 * vector. 580 */ 581 err = mlx5_vector2eqn(mdev, 0, &eqn); 582 if (err) 583 goto err_vec; 584 585 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 586 MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent)); 587 MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); 588 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 589 MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma); 590 591 err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out)); 592 if (err) 593 goto err_vec; 594 595 vcq->mcq.comp = mlx5_vdpa_cq_comp; 596 vcq->cqe = num_ent; 597 vcq->mcq.set_ci_db = vcq->db.db; 598 vcq->mcq.arm_db = vcq->db.db + 1; 599 mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 600 kfree(in); 601 return 0; 602 603 err_vec: 604 kfree(in); 605 err_vzalloc: 606 cq_frag_buf_free(ndev, &vcq->buf); 607 err_db: 608 mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 609 return err; 610 } 611 612 static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) 613 { 614 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 615 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 616 struct mlx5_vdpa_cq *vcq = &mvq->cq; 617 618 if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) { 619 mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); 620 return; 621 } 622 cq_frag_buf_free(ndev, &vcq->buf); 623 mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 624 } 625 626 static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, 627 struct mlx5_vdpa_umem **umemp) 628 { 629 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 630 int p_a; 631 int p_b; 632 633 switch (num) { 634 case 1: 635 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a); 636 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b); 637 *umemp = &mvq->umem1; 638 break; 639 case 2: 640 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a); 641 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b); 642 *umemp = &mvq->umem2; 643 break; 644 case 3: 645 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a); 646 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b); 647 *umemp = &mvq->umem3; 648 break; 649 } 650 (*umemp)->size = p_a * mvq->num_ent + p_b; 651 } 652 653 static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) 654 { 655 mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); 656 } 657 658 static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 659 { 660 int inlen; 661 u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {}; 662 void *um; 663 void *in; 664 int err; 665 __be64 *pas; 666 struct mlx5_vdpa_umem *umem; 667 668 set_umem_size(ndev, mvq, num, &umem); 669 err = umem_frag_buf_alloc(ndev, umem, umem->size); 670 if (err) 671 return err; 672 673 inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages; 674 675 in = kzalloc(inlen, GFP_KERNEL); 676 if (!in) { 677 err = -ENOMEM; 678 goto err_in; 679 } 680 681 MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM); 682 MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); 683 um = MLX5_ADDR_OF(create_umem_in, in, umem); 684 MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 685 MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages); 686 687 pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]); 688 mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW); 689 690 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 691 if (err) { 692 mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); 693 goto err_cmd; 694 } 695 696 kfree(in); 697 umem->id = MLX5_GET(create_umem_out, out, umem_id); 698 699 return 0; 700 701 err_cmd: 702 kfree(in); 703 err_in: 704 umem_frag_buf_free(ndev, umem); 705 return err; 706 } 707 708 static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 709 { 710 u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {}; 711 u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {}; 712 struct mlx5_vdpa_umem *umem; 713 714 switch (num) { 715 case 1: 716 umem = &mvq->umem1; 717 break; 718 case 2: 719 umem = &mvq->umem2; 720 break; 721 case 3: 722 umem = &mvq->umem3; 723 break; 724 } 725 726 MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM); 727 MLX5_SET(destroy_umem_in, in, umem_id, umem->id); 728 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 729 return; 730 731 umem_frag_buf_free(ndev, umem); 732 } 733 734 static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 735 { 736 int num; 737 int err; 738 739 for (num = 1; num <= 3; num++) { 740 err = create_umem(ndev, mvq, num); 741 if (err) 742 goto err_umem; 743 } 744 return 0; 745 746 err_umem: 747 for (num--; num > 0; num--) 748 umem_destroy(ndev, mvq, num); 749 750 return err; 751 } 752 753 static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 754 { 755 int num; 756 757 for (num = 3; num > 0; num--) 758 umem_destroy(ndev, mvq, num); 759 } 760 761 static int get_queue_type(struct mlx5_vdpa_net *ndev) 762 { 763 u32 type_mask; 764 765 type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); 766 767 /* prefer split queue */ 768 if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT) 769 return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT; 770 771 WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED)); 772 773 return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED; 774 } 775 776 static bool vq_is_tx(u16 idx) 777 { 778 return idx % 2; 779 } 780 781 static u16 get_features_12_3(u64 features) 782 { 783 return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) | 784 (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) | 785 (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) | 786 (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6); 787 } 788 789 static bool counters_supported(const struct mlx5_vdpa_dev *mvdev) 790 { 791 return MLX5_CAP_GEN_64(mvdev->mdev, general_obj_types) & 792 BIT_ULL(MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 793 } 794 795 static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 796 { 797 int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); 798 u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {}; 799 void *obj_context; 800 void *cmd_hdr; 801 void *vq_ctx; 802 void *in; 803 int err; 804 805 err = umems_create(ndev, mvq); 806 if (err) 807 return err; 808 809 in = kzalloc(inlen, GFP_KERNEL); 810 if (!in) { 811 err = -ENOMEM; 812 goto err_alloc; 813 } 814 815 cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr); 816 817 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 818 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 819 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 820 821 obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context); 822 MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); 823 MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); 824 MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, 825 get_features_12_3(ndev->mvdev.actual_features)); 826 vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context); 827 MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); 828 829 if (vq_is_tx(mvq->index)) 830 MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); 831 832 MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 833 MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); 834 MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 835 MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); 836 MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 837 !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); 838 MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); 839 MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); 840 MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); 841 MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey); 842 MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); 843 MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); 844 MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id); 845 MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem2.size); 846 MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id); 847 MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem3.size); 848 MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); 849 if (counters_supported(&ndev->mvdev)) 850 MLX5_SET(virtio_q, vq_ctx, counter_set_id, mvq->counter_set_id); 851 852 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 853 if (err) 854 goto err_cmd; 855 856 mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT; 857 kfree(in); 858 mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 859 860 return 0; 861 862 err_cmd: 863 kfree(in); 864 err_alloc: 865 umems_destroy(ndev, mvq); 866 return err; 867 } 868 869 static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 870 { 871 u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {}; 872 u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {}; 873 874 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode, 875 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 876 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id); 877 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); 878 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type, 879 MLX5_OBJ_TYPE_VIRTIO_NET_Q); 880 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { 881 mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); 882 return; 883 } 884 mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; 885 umems_destroy(ndev, mvq); 886 } 887 888 static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 889 { 890 return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn; 891 } 892 893 static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 894 { 895 return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn; 896 } 897 898 static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, 899 int *outlen, u32 qpn, u32 rqpn) 900 { 901 void *qpc; 902 void *pp; 903 904 switch (cmd) { 905 case MLX5_CMD_OP_2RST_QP: 906 *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in); 907 *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out); 908 *in = kzalloc(*inlen, GFP_KERNEL); 909 *out = kzalloc(*outlen, GFP_KERNEL); 910 if (!*in || !*out) 911 goto outerr; 912 913 MLX5_SET(qp_2rst_in, *in, opcode, cmd); 914 MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); 915 MLX5_SET(qp_2rst_in, *in, qpn, qpn); 916 break; 917 case MLX5_CMD_OP_RST2INIT_QP: 918 *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in); 919 *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out); 920 *in = kzalloc(*inlen, GFP_KERNEL); 921 *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL); 922 if (!*in || !*out) 923 goto outerr; 924 925 MLX5_SET(rst2init_qp_in, *in, opcode, cmd); 926 MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); 927 MLX5_SET(rst2init_qp_in, *in, qpn, qpn); 928 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 929 MLX5_SET(qpc, qpc, remote_qpn, rqpn); 930 MLX5_SET(qpc, qpc, rwe, 1); 931 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 932 MLX5_SET(ads, pp, vhca_port_num, 1); 933 break; 934 case MLX5_CMD_OP_INIT2RTR_QP: 935 *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in); 936 *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out); 937 *in = kzalloc(*inlen, GFP_KERNEL); 938 *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL); 939 if (!*in || !*out) 940 goto outerr; 941 942 MLX5_SET(init2rtr_qp_in, *in, opcode, cmd); 943 MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); 944 MLX5_SET(init2rtr_qp_in, *in, qpn, qpn); 945 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 946 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 947 MLX5_SET(qpc, qpc, log_msg_max, 30); 948 MLX5_SET(qpc, qpc, remote_qpn, rqpn); 949 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 950 MLX5_SET(ads, pp, fl, 1); 951 break; 952 case MLX5_CMD_OP_RTR2RTS_QP: 953 *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in); 954 *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out); 955 *in = kzalloc(*inlen, GFP_KERNEL); 956 *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL); 957 if (!*in || !*out) 958 goto outerr; 959 960 MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd); 961 MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); 962 MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn); 963 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 964 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 965 MLX5_SET(ads, pp, ack_timeout, 14); 966 MLX5_SET(qpc, qpc, retry_count, 7); 967 MLX5_SET(qpc, qpc, rnr_retry, 7); 968 break; 969 default: 970 goto outerr_nullify; 971 } 972 973 return; 974 975 outerr: 976 kfree(*in); 977 kfree(*out); 978 outerr_nullify: 979 *in = NULL; 980 *out = NULL; 981 } 982 983 static void free_inout(void *in, void *out) 984 { 985 kfree(in); 986 kfree(out); 987 } 988 989 /* Two QPs are used by each virtqueue. One is used by the driver and one by 990 * firmware. The fw argument indicates whether the subjected QP is the one used 991 * by firmware. 992 */ 993 static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) 994 { 995 int outlen; 996 int inlen; 997 void *out; 998 void *in; 999 int err; 1000 1001 alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); 1002 if (!in || !out) 1003 return -ENOMEM; 1004 1005 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); 1006 free_inout(in, out); 1007 return err; 1008 } 1009 1010 static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1011 { 1012 int err; 1013 1014 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); 1015 if (err) 1016 return err; 1017 1018 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); 1019 if (err) 1020 return err; 1021 1022 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); 1023 if (err) 1024 return err; 1025 1026 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); 1027 if (err) 1028 return err; 1029 1030 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); 1031 if (err) 1032 return err; 1033 1034 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); 1035 if (err) 1036 return err; 1037 1038 return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); 1039 } 1040 1041 struct mlx5_virtq_attr { 1042 u8 state; 1043 u16 available_index; 1044 u16 used_index; 1045 }; 1046 1047 static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 1048 struct mlx5_virtq_attr *attr) 1049 { 1050 int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out); 1051 u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {}; 1052 void *out; 1053 void *obj_context; 1054 void *cmd_hdr; 1055 int err; 1056 1057 out = kzalloc(outlen, GFP_KERNEL); 1058 if (!out) 1059 return -ENOMEM; 1060 1061 cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr); 1062 1063 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 1064 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 1065 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 1066 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 1067 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen); 1068 if (err) 1069 goto err_cmd; 1070 1071 obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context); 1072 memset(attr, 0, sizeof(*attr)); 1073 attr->state = MLX5_GET(virtio_net_q_object, obj_context, state); 1074 attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index); 1075 attr->used_index = MLX5_GET(virtio_net_q_object, obj_context, hw_used_index); 1076 kfree(out); 1077 return 0; 1078 1079 err_cmd: 1080 kfree(out); 1081 return err; 1082 } 1083 1084 static bool is_valid_state_change(int oldstate, int newstate) 1085 { 1086 switch (oldstate) { 1087 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT: 1088 return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY; 1089 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY: 1090 return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND; 1091 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND: 1092 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR: 1093 default: 1094 return false; 1095 } 1096 } 1097 1098 static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) 1099 { 1100 int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); 1101 u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {}; 1102 void *obj_context; 1103 void *cmd_hdr; 1104 void *in; 1105 int err; 1106 1107 if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_NONE) 1108 return 0; 1109 1110 if (!is_valid_state_change(mvq->fw_state, state)) 1111 return -EINVAL; 1112 1113 in = kzalloc(inlen, GFP_KERNEL); 1114 if (!in) 1115 return -ENOMEM; 1116 1117 cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr); 1118 1119 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); 1120 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 1121 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 1122 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 1123 1124 obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context); 1125 MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select, 1126 MLX5_VIRTQ_MODIFY_MASK_STATE); 1127 MLX5_SET(virtio_net_q_object, obj_context, state, state); 1128 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 1129 kfree(in); 1130 if (!err) 1131 mvq->fw_state = state; 1132 1133 return err; 1134 } 1135 1136 static int counter_set_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1137 { 1138 u32 in[MLX5_ST_SZ_DW(create_virtio_q_counters_in)] = {}; 1139 u32 out[MLX5_ST_SZ_DW(create_virtio_q_counters_out)] = {}; 1140 void *cmd_hdr; 1141 int err; 1142 1143 if (!counters_supported(&ndev->mvdev)) 1144 return 0; 1145 1146 cmd_hdr = MLX5_ADDR_OF(create_virtio_q_counters_in, in, hdr); 1147 1148 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 1149 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 1150 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 1151 1152 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 1153 if (err) 1154 return err; 1155 1156 mvq->counter_set_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 1157 1158 return 0; 1159 } 1160 1161 static void counter_set_dealloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1162 { 1163 u32 in[MLX5_ST_SZ_DW(destroy_virtio_q_counters_in)] = {}; 1164 u32 out[MLX5_ST_SZ_DW(destroy_virtio_q_counters_out)] = {}; 1165 1166 if (!counters_supported(&ndev->mvdev)) 1167 return; 1168 1169 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 1170 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_id, mvq->counter_set_id); 1171 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.uid, ndev->mvdev.res.uid); 1172 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 1173 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 1174 mlx5_vdpa_warn(&ndev->mvdev, "dealloc counter set 0x%x\n", mvq->counter_set_id); 1175 } 1176 1177 static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1178 { 1179 u16 idx = mvq->index; 1180 int err; 1181 1182 if (!mvq->num_ent) 1183 return 0; 1184 1185 if (mvq->initialized) 1186 return 0; 1187 1188 err = cq_create(ndev, idx, mvq->num_ent); 1189 if (err) 1190 return err; 1191 1192 err = qp_create(ndev, mvq, &mvq->fwqp); 1193 if (err) 1194 goto err_fwqp; 1195 1196 err = qp_create(ndev, mvq, &mvq->vqqp); 1197 if (err) 1198 goto err_vqqp; 1199 1200 err = connect_qps(ndev, mvq); 1201 if (err) 1202 goto err_connect; 1203 1204 err = counter_set_alloc(ndev, mvq); 1205 if (err) 1206 goto err_counter; 1207 1208 err = create_virtqueue(ndev, mvq); 1209 if (err) 1210 goto err_connect; 1211 1212 if (mvq->ready) { 1213 err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 1214 if (err) { 1215 mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", 1216 idx, err); 1217 goto err_connect; 1218 } 1219 } 1220 1221 mvq->initialized = true; 1222 return 0; 1223 1224 err_connect: 1225 counter_set_dealloc(ndev, mvq); 1226 err_counter: 1227 qp_destroy(ndev, &mvq->vqqp); 1228 err_vqqp: 1229 qp_destroy(ndev, &mvq->fwqp); 1230 err_fwqp: 1231 cq_destroy(ndev, idx); 1232 return err; 1233 } 1234 1235 static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1236 { 1237 struct mlx5_virtq_attr attr; 1238 1239 if (!mvq->initialized) 1240 return; 1241 1242 if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 1243 return; 1244 1245 if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) 1246 mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); 1247 1248 if (query_virtqueue(ndev, mvq, &attr)) { 1249 mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); 1250 return; 1251 } 1252 mvq->avail_idx = attr.available_index; 1253 mvq->used_idx = attr.used_index; 1254 } 1255 1256 static void suspend_vqs(struct mlx5_vdpa_net *ndev) 1257 { 1258 int i; 1259 1260 for (i = 0; i < ndev->mvdev.max_vqs; i++) 1261 suspend_vq(ndev, &ndev->vqs[i]); 1262 } 1263 1264 static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1265 { 1266 if (!mvq->initialized) 1267 return; 1268 1269 suspend_vq(ndev, mvq); 1270 destroy_virtqueue(ndev, mvq); 1271 counter_set_dealloc(ndev, mvq); 1272 qp_destroy(ndev, &mvq->vqqp); 1273 qp_destroy(ndev, &mvq->fwqp); 1274 cq_destroy(ndev, mvq->index); 1275 mvq->initialized = false; 1276 } 1277 1278 static int create_rqt(struct mlx5_vdpa_net *ndev) 1279 { 1280 int rqt_table_size = roundup_pow_of_two(ndev->rqt_size); 1281 int act_sz = roundup_pow_of_two(ndev->cur_num_vqs / 2); 1282 __be32 *list; 1283 void *rqtc; 1284 int inlen; 1285 void *in; 1286 int i, j; 1287 int err; 1288 1289 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + rqt_table_size * MLX5_ST_SZ_BYTES(rq_num); 1290 in = kzalloc(inlen, GFP_KERNEL); 1291 if (!in) 1292 return -ENOMEM; 1293 1294 MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); 1295 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 1296 1297 MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 1298 MLX5_SET(rqtc, rqtc, rqt_max_size, rqt_table_size); 1299 list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1300 for (i = 0, j = 0; i < act_sz; i++, j += 2) 1301 list[i] = cpu_to_be32(ndev->vqs[j % ndev->cur_num_vqs].virtq_id); 1302 1303 MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz); 1304 err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); 1305 kfree(in); 1306 if (err) 1307 return err; 1308 1309 return 0; 1310 } 1311 1312 #define MLX5_MODIFY_RQT_NUM_RQS ((u64)1) 1313 1314 static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) 1315 { 1316 int act_sz = roundup_pow_of_two(num / 2); 1317 __be32 *list; 1318 void *rqtc; 1319 int inlen; 1320 void *in; 1321 int i, j; 1322 int err; 1323 1324 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + act_sz * MLX5_ST_SZ_BYTES(rq_num); 1325 in = kzalloc(inlen, GFP_KERNEL); 1326 if (!in) 1327 return -ENOMEM; 1328 1329 MLX5_SET(modify_rqt_in, in, uid, ndev->mvdev.res.uid); 1330 MLX5_SET64(modify_rqt_in, in, bitmask, MLX5_MODIFY_RQT_NUM_RQS); 1331 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 1332 MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 1333 1334 list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1335 for (i = 0, j = 0; i < act_sz; i++, j = j + 2) 1336 list[i] = cpu_to_be32(ndev->vqs[j % num].virtq_id); 1337 1338 MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz); 1339 err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn); 1340 kfree(in); 1341 if (err) 1342 return err; 1343 1344 return 0; 1345 } 1346 1347 static void destroy_rqt(struct mlx5_vdpa_net *ndev) 1348 { 1349 mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); 1350 } 1351 1352 static int create_tir(struct mlx5_vdpa_net *ndev) 1353 { 1354 #define HASH_IP_L4PORTS \ 1355 (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \ 1356 MLX5_HASH_FIELD_SEL_L4_DPORT) 1357 static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 1358 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 1359 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 1360 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 1361 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a }; 1362 void *rss_key; 1363 void *outer; 1364 void *tirc; 1365 void *in; 1366 int err; 1367 1368 in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL); 1369 if (!in) 1370 return -ENOMEM; 1371 1372 MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); 1373 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 1374 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 1375 1376 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 1377 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ); 1378 rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 1379 memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key)); 1380 1381 outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 1382 MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); 1383 MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); 1384 MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS); 1385 1386 MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); 1387 MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); 1388 1389 err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); 1390 kfree(in); 1391 if (err) 1392 return err; 1393 1394 mlx5_vdpa_add_tirn(ndev); 1395 return err; 1396 } 1397 1398 static void destroy_tir(struct mlx5_vdpa_net *ndev) 1399 { 1400 mlx5_vdpa_remove_tirn(ndev); 1401 mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); 1402 } 1403 1404 #define MAX_STEERING_ENT 0x8000 1405 #define MAX_STEERING_GROUPS 2 1406 1407 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1408 #define NUM_DESTS 2 1409 #else 1410 #define NUM_DESTS 1 1411 #endif 1412 1413 static int add_steering_counters(struct mlx5_vdpa_net *ndev, 1414 struct macvlan_node *node, 1415 struct mlx5_flow_act *flow_act, 1416 struct mlx5_flow_destination *dests) 1417 { 1418 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1419 int err; 1420 1421 node->ucast_counter.counter = mlx5_fc_create(ndev->mvdev.mdev, false); 1422 if (IS_ERR(node->ucast_counter.counter)) 1423 return PTR_ERR(node->ucast_counter.counter); 1424 1425 node->mcast_counter.counter = mlx5_fc_create(ndev->mvdev.mdev, false); 1426 if (IS_ERR(node->mcast_counter.counter)) { 1427 err = PTR_ERR(node->mcast_counter.counter); 1428 goto err_mcast_counter; 1429 } 1430 1431 dests[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1432 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 1433 return 0; 1434 1435 err_mcast_counter: 1436 mlx5_fc_destroy(ndev->mvdev.mdev, node->ucast_counter.counter); 1437 return err; 1438 #else 1439 return 0; 1440 #endif 1441 } 1442 1443 static void remove_steering_counters(struct mlx5_vdpa_net *ndev, 1444 struct macvlan_node *node) 1445 { 1446 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1447 mlx5_fc_destroy(ndev->mvdev.mdev, node->mcast_counter.counter); 1448 mlx5_fc_destroy(ndev->mvdev.mdev, node->ucast_counter.counter); 1449 #endif 1450 } 1451 1452 static int mlx5_vdpa_add_mac_vlan_rules(struct mlx5_vdpa_net *ndev, u8 *mac, 1453 struct macvlan_node *node) 1454 { 1455 struct mlx5_flow_destination dests[NUM_DESTS] = {}; 1456 struct mlx5_flow_act flow_act = {}; 1457 struct mlx5_flow_spec *spec; 1458 void *headers_c; 1459 void *headers_v; 1460 u8 *dmac_c; 1461 u8 *dmac_v; 1462 int err; 1463 u16 vid; 1464 1465 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1466 if (!spec) 1467 return -ENOMEM; 1468 1469 vid = key2vid(node->macvlan); 1470 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1471 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers); 1472 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); 1473 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, outer_headers.dmac_47_16); 1474 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16); 1475 eth_broadcast_addr(dmac_c); 1476 ether_addr_copy(dmac_v, mac); 1477 if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VLAN)) { 1478 MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1); 1479 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, first_vid); 1480 } 1481 if (node->tagged) { 1482 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); 1483 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, vid); 1484 } 1485 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1486 dests[0].type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1487 dests[0].tir_num = ndev->res.tirn; 1488 err = add_steering_counters(ndev, node, &flow_act, dests); 1489 if (err) 1490 goto out_free; 1491 1492 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1493 dests[1].counter_id = mlx5_fc_id(node->ucast_counter.counter); 1494 #endif 1495 node->ucast_rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dests, NUM_DESTS); 1496 if (IS_ERR(node->ucast_rule)) { 1497 err = PTR_ERR(node->ucast_rule); 1498 goto err_ucast; 1499 } 1500 1501 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1502 dests[1].counter_id = mlx5_fc_id(node->mcast_counter.counter); 1503 #endif 1504 1505 memset(dmac_c, 0, ETH_ALEN); 1506 memset(dmac_v, 0, ETH_ALEN); 1507 dmac_c[0] = 1; 1508 dmac_v[0] = 1; 1509 node->mcast_rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dests, NUM_DESTS); 1510 if (IS_ERR(node->mcast_rule)) { 1511 err = PTR_ERR(node->mcast_rule); 1512 goto err_mcast; 1513 } 1514 kvfree(spec); 1515 mlx5_vdpa_add_rx_counters(ndev, node); 1516 return 0; 1517 1518 err_mcast: 1519 mlx5_del_flow_rules(node->ucast_rule); 1520 err_ucast: 1521 remove_steering_counters(ndev, node); 1522 out_free: 1523 kvfree(spec); 1524 return err; 1525 } 1526 1527 static void mlx5_vdpa_del_mac_vlan_rules(struct mlx5_vdpa_net *ndev, 1528 struct macvlan_node *node) 1529 { 1530 mlx5_vdpa_remove_rx_counters(ndev, node); 1531 mlx5_del_flow_rules(node->ucast_rule); 1532 mlx5_del_flow_rules(node->mcast_rule); 1533 } 1534 1535 static u64 search_val(u8 *mac, u16 vlan, bool tagged) 1536 { 1537 u64 val; 1538 1539 if (!tagged) 1540 vlan = MLX5V_UNTAGGED; 1541 1542 val = (u64)vlan << 48 | 1543 (u64)mac[0] << 40 | 1544 (u64)mac[1] << 32 | 1545 (u64)mac[2] << 24 | 1546 (u64)mac[3] << 16 | 1547 (u64)mac[4] << 8 | 1548 (u64)mac[5]; 1549 1550 return val; 1551 } 1552 1553 static struct macvlan_node *mac_vlan_lookup(struct mlx5_vdpa_net *ndev, u64 value) 1554 { 1555 struct macvlan_node *pos; 1556 u32 idx; 1557 1558 idx = hash_64(value, 8); // tbd 8 1559 hlist_for_each_entry(pos, &ndev->macvlan_hash[idx], hlist) { 1560 if (pos->macvlan == value) 1561 return pos; 1562 } 1563 return NULL; 1564 } 1565 1566 static int mac_vlan_add(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vid, bool tagged) 1567 { 1568 struct macvlan_node *ptr; 1569 u64 val; 1570 u32 idx; 1571 int err; 1572 1573 val = search_val(mac, vid, tagged); 1574 if (mac_vlan_lookup(ndev, val)) 1575 return -EEXIST; 1576 1577 ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); 1578 if (!ptr) 1579 return -ENOMEM; 1580 1581 ptr->tagged = tagged; 1582 ptr->macvlan = val; 1583 ptr->ndev = ndev; 1584 err = mlx5_vdpa_add_mac_vlan_rules(ndev, ndev->config.mac, ptr); 1585 if (err) 1586 goto err_add; 1587 1588 idx = hash_64(val, 8); 1589 hlist_add_head(&ptr->hlist, &ndev->macvlan_hash[idx]); 1590 return 0; 1591 1592 err_add: 1593 kfree(ptr); 1594 return err; 1595 } 1596 1597 static void mac_vlan_del(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vlan, bool tagged) 1598 { 1599 struct macvlan_node *ptr; 1600 1601 ptr = mac_vlan_lookup(ndev, search_val(mac, vlan, tagged)); 1602 if (!ptr) 1603 return; 1604 1605 hlist_del(&ptr->hlist); 1606 mlx5_vdpa_del_mac_vlan_rules(ndev, ptr); 1607 remove_steering_counters(ndev, ptr); 1608 kfree(ptr); 1609 } 1610 1611 static void clear_mac_vlan_table(struct mlx5_vdpa_net *ndev) 1612 { 1613 struct macvlan_node *pos; 1614 struct hlist_node *n; 1615 int i; 1616 1617 for (i = 0; i < MLX5V_MACVLAN_SIZE; i++) { 1618 hlist_for_each_entry_safe(pos, n, &ndev->macvlan_hash[i], hlist) { 1619 hlist_del(&pos->hlist); 1620 mlx5_vdpa_del_mac_vlan_rules(ndev, pos); 1621 remove_steering_counters(ndev, pos); 1622 kfree(pos); 1623 } 1624 } 1625 } 1626 1627 static int setup_steering(struct mlx5_vdpa_net *ndev) 1628 { 1629 struct mlx5_flow_table_attr ft_attr = {}; 1630 struct mlx5_flow_namespace *ns; 1631 int err; 1632 1633 ft_attr.max_fte = MAX_STEERING_ENT; 1634 ft_attr.autogroup.max_num_groups = MAX_STEERING_GROUPS; 1635 1636 ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); 1637 if (!ns) { 1638 mlx5_vdpa_warn(&ndev->mvdev, "failed to get flow namespace\n"); 1639 return -EOPNOTSUPP; 1640 } 1641 1642 ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 1643 if (IS_ERR(ndev->rxft)) { 1644 mlx5_vdpa_warn(&ndev->mvdev, "failed to create flow table\n"); 1645 return PTR_ERR(ndev->rxft); 1646 } 1647 mlx5_vdpa_add_rx_flow_table(ndev); 1648 1649 err = mac_vlan_add(ndev, ndev->config.mac, 0, false); 1650 if (err) 1651 goto err_add; 1652 1653 return 0; 1654 1655 err_add: 1656 mlx5_vdpa_remove_rx_flow_table(ndev); 1657 mlx5_destroy_flow_table(ndev->rxft); 1658 return err; 1659 } 1660 1661 static void teardown_steering(struct mlx5_vdpa_net *ndev) 1662 { 1663 clear_mac_vlan_table(ndev); 1664 mlx5_vdpa_remove_rx_flow_table(ndev); 1665 mlx5_destroy_flow_table(ndev->rxft); 1666 } 1667 1668 static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1669 { 1670 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1671 struct mlx5_control_vq *cvq = &mvdev->cvq; 1672 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1673 struct mlx5_core_dev *pfmdev; 1674 size_t read; 1675 u8 mac[ETH_ALEN], mac_back[ETH_ALEN]; 1676 1677 pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 1678 switch (cmd) { 1679 case VIRTIO_NET_CTRL_MAC_ADDR_SET: 1680 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)mac, ETH_ALEN); 1681 if (read != ETH_ALEN) 1682 break; 1683 1684 if (!memcmp(ndev->config.mac, mac, 6)) { 1685 status = VIRTIO_NET_OK; 1686 break; 1687 } 1688 1689 if (is_zero_ether_addr(mac)) 1690 break; 1691 1692 if (!is_zero_ether_addr(ndev->config.mac)) { 1693 if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 1694 mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n", 1695 ndev->config.mac); 1696 break; 1697 } 1698 } 1699 1700 if (mlx5_mpfs_add_mac(pfmdev, mac)) { 1701 mlx5_vdpa_warn(mvdev, "failed to insert new MAC %pM into MPFS table\n", 1702 mac); 1703 break; 1704 } 1705 1706 /* backup the original mac address so that if failed to add the forward rules 1707 * we could restore it 1708 */ 1709 memcpy(mac_back, ndev->config.mac, ETH_ALEN); 1710 1711 memcpy(ndev->config.mac, mac, ETH_ALEN); 1712 1713 /* Need recreate the flow table entry, so that the packet could forward back 1714 */ 1715 mac_vlan_del(ndev, mac_back, 0, false); 1716 1717 if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) { 1718 mlx5_vdpa_warn(mvdev, "failed to insert forward rules, try to restore\n"); 1719 1720 /* Although it hardly run here, we still need double check */ 1721 if (is_zero_ether_addr(mac_back)) { 1722 mlx5_vdpa_warn(mvdev, "restore mac failed: Original MAC is zero\n"); 1723 break; 1724 } 1725 1726 /* Try to restore original mac address to MFPS table, and try to restore 1727 * the forward rule entry. 1728 */ 1729 if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 1730 mlx5_vdpa_warn(mvdev, "restore mac failed: delete MAC %pM from MPFS table failed\n", 1731 ndev->config.mac); 1732 } 1733 1734 if (mlx5_mpfs_add_mac(pfmdev, mac_back)) { 1735 mlx5_vdpa_warn(mvdev, "restore mac failed: insert old MAC %pM into MPFS table failed\n", 1736 mac_back); 1737 } 1738 1739 memcpy(ndev->config.mac, mac_back, ETH_ALEN); 1740 1741 if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) 1742 mlx5_vdpa_warn(mvdev, "restore forward rules failed: insert forward rules failed\n"); 1743 1744 break; 1745 } 1746 1747 status = VIRTIO_NET_OK; 1748 break; 1749 1750 default: 1751 break; 1752 } 1753 1754 return status; 1755 } 1756 1757 static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps) 1758 { 1759 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1760 int cur_qps = ndev->cur_num_vqs / 2; 1761 int err; 1762 int i; 1763 1764 if (cur_qps > newqps) { 1765 err = modify_rqt(ndev, 2 * newqps); 1766 if (err) 1767 return err; 1768 1769 for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) 1770 teardown_vq(ndev, &ndev->vqs[i]); 1771 1772 ndev->cur_num_vqs = 2 * newqps; 1773 } else { 1774 ndev->cur_num_vqs = 2 * newqps; 1775 for (i = cur_qps * 2; i < 2 * newqps; i++) { 1776 err = setup_vq(ndev, &ndev->vqs[i]); 1777 if (err) 1778 goto clean_added; 1779 } 1780 err = modify_rqt(ndev, 2 * newqps); 1781 if (err) 1782 goto clean_added; 1783 } 1784 return 0; 1785 1786 clean_added: 1787 for (--i; i >= 2 * cur_qps; --i) 1788 teardown_vq(ndev, &ndev->vqs[i]); 1789 1790 ndev->cur_num_vqs = 2 * cur_qps; 1791 1792 return err; 1793 } 1794 1795 static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1796 { 1797 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1798 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1799 struct mlx5_control_vq *cvq = &mvdev->cvq; 1800 struct virtio_net_ctrl_mq mq; 1801 size_t read; 1802 u16 newqps; 1803 1804 switch (cmd) { 1805 case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: 1806 /* This mq feature check aligns with pre-existing userspace 1807 * implementation. 1808 * 1809 * Without it, an untrusted driver could fake a multiqueue config 1810 * request down to a non-mq device that may cause kernel to 1811 * panic due to uninitialized resources for extra vqs. Even with 1812 * a well behaving guest driver, it is not expected to allow 1813 * changing the number of vqs on a non-mq device. 1814 */ 1815 if (!MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) 1816 break; 1817 1818 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)&mq, sizeof(mq)); 1819 if (read != sizeof(mq)) 1820 break; 1821 1822 newqps = mlx5vdpa16_to_cpu(mvdev, mq.virtqueue_pairs); 1823 if (newqps < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 1824 newqps > ndev->rqt_size) 1825 break; 1826 1827 if (ndev->cur_num_vqs == 2 * newqps) { 1828 status = VIRTIO_NET_OK; 1829 break; 1830 } 1831 1832 if (!change_num_qps(mvdev, newqps)) 1833 status = VIRTIO_NET_OK; 1834 1835 break; 1836 default: 1837 break; 1838 } 1839 1840 return status; 1841 } 1842 1843 static virtio_net_ctrl_ack handle_ctrl_vlan(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1844 { 1845 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1846 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1847 struct mlx5_control_vq *cvq = &mvdev->cvq; 1848 __virtio16 vlan; 1849 size_t read; 1850 u16 id; 1851 1852 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VLAN))) 1853 return status; 1854 1855 switch (cmd) { 1856 case VIRTIO_NET_CTRL_VLAN_ADD: 1857 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &vlan, sizeof(vlan)); 1858 if (read != sizeof(vlan)) 1859 break; 1860 1861 id = mlx5vdpa16_to_cpu(mvdev, vlan); 1862 if (mac_vlan_add(ndev, ndev->config.mac, id, true)) 1863 break; 1864 1865 status = VIRTIO_NET_OK; 1866 break; 1867 case VIRTIO_NET_CTRL_VLAN_DEL: 1868 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &vlan, sizeof(vlan)); 1869 if (read != sizeof(vlan)) 1870 break; 1871 1872 id = mlx5vdpa16_to_cpu(mvdev, vlan); 1873 mac_vlan_del(ndev, ndev->config.mac, id, true); 1874 status = VIRTIO_NET_OK; 1875 break; 1876 default: 1877 break; 1878 } 1879 1880 return status; 1881 } 1882 1883 static void mlx5_cvq_kick_handler(struct work_struct *work) 1884 { 1885 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1886 struct virtio_net_ctrl_hdr ctrl; 1887 struct mlx5_vdpa_wq_ent *wqent; 1888 struct mlx5_vdpa_dev *mvdev; 1889 struct mlx5_control_vq *cvq; 1890 struct mlx5_vdpa_net *ndev; 1891 size_t read, write; 1892 int err; 1893 1894 wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 1895 mvdev = wqent->mvdev; 1896 ndev = to_mlx5_vdpa_ndev(mvdev); 1897 cvq = &mvdev->cvq; 1898 1899 down_write(&ndev->reslock); 1900 1901 if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 1902 goto out; 1903 1904 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 1905 goto out; 1906 1907 if (!cvq->ready) 1908 goto out; 1909 1910 while (true) { 1911 err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head, 1912 GFP_ATOMIC); 1913 if (err <= 0) 1914 break; 1915 1916 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &ctrl, sizeof(ctrl)); 1917 if (read != sizeof(ctrl)) 1918 break; 1919 1920 cvq->received_desc++; 1921 switch (ctrl.class) { 1922 case VIRTIO_NET_CTRL_MAC: 1923 status = handle_ctrl_mac(mvdev, ctrl.cmd); 1924 break; 1925 case VIRTIO_NET_CTRL_MQ: 1926 status = handle_ctrl_mq(mvdev, ctrl.cmd); 1927 break; 1928 case VIRTIO_NET_CTRL_VLAN: 1929 status = handle_ctrl_vlan(mvdev, ctrl.cmd); 1930 break; 1931 default: 1932 break; 1933 } 1934 1935 /* Make sure data is written before advancing index */ 1936 smp_wmb(); 1937 1938 write = vringh_iov_push_iotlb(&cvq->vring, &cvq->wiov, &status, sizeof(status)); 1939 vringh_complete_iotlb(&cvq->vring, cvq->head, write); 1940 vringh_kiov_cleanup(&cvq->riov); 1941 vringh_kiov_cleanup(&cvq->wiov); 1942 1943 if (vringh_need_notify_iotlb(&cvq->vring)) 1944 vringh_notify(&cvq->vring); 1945 1946 cvq->completed_desc++; 1947 queue_work(mvdev->wq, &wqent->work); 1948 break; 1949 } 1950 1951 out: 1952 up_write(&ndev->reslock); 1953 } 1954 1955 static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) 1956 { 1957 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 1958 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1959 struct mlx5_vdpa_virtqueue *mvq; 1960 1961 if (!is_index_valid(mvdev, idx)) 1962 return; 1963 1964 if (unlikely(is_ctrl_vq_idx(mvdev, idx))) { 1965 if (!mvdev->wq || !mvdev->cvq.ready) 1966 return; 1967 1968 queue_work(mvdev->wq, &ndev->cvq_ent.work); 1969 return; 1970 } 1971 1972 mvq = &ndev->vqs[idx]; 1973 if (unlikely(!mvq->ready)) 1974 return; 1975 1976 iowrite16(idx, ndev->mvdev.res.kick_addr); 1977 } 1978 1979 static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 1980 u64 driver_area, u64 device_area) 1981 { 1982 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 1983 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1984 struct mlx5_vdpa_virtqueue *mvq; 1985 1986 if (!is_index_valid(mvdev, idx)) 1987 return -EINVAL; 1988 1989 if (is_ctrl_vq_idx(mvdev, idx)) { 1990 mvdev->cvq.desc_addr = desc_area; 1991 mvdev->cvq.device_addr = device_area; 1992 mvdev->cvq.driver_addr = driver_area; 1993 return 0; 1994 } 1995 1996 mvq = &ndev->vqs[idx]; 1997 mvq->desc_addr = desc_area; 1998 mvq->device_addr = device_area; 1999 mvq->driver_addr = driver_area; 2000 return 0; 2001 } 2002 2003 static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 2004 { 2005 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2006 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2007 struct mlx5_vdpa_virtqueue *mvq; 2008 2009 if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2010 return; 2011 2012 mvq = &ndev->vqs[idx]; 2013 mvq->num_ent = num; 2014 } 2015 2016 static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 2017 { 2018 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2019 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2020 2021 ndev->event_cbs[idx] = *cb; 2022 if (is_ctrl_vq_idx(mvdev, idx)) 2023 mvdev->cvq.event_cb = *cb; 2024 } 2025 2026 static void mlx5_cvq_notify(struct vringh *vring) 2027 { 2028 struct mlx5_control_vq *cvq = container_of(vring, struct mlx5_control_vq, vring); 2029 2030 if (!cvq->event_cb.callback) 2031 return; 2032 2033 cvq->event_cb.callback(cvq->event_cb.private); 2034 } 2035 2036 static void set_cvq_ready(struct mlx5_vdpa_dev *mvdev, bool ready) 2037 { 2038 struct mlx5_control_vq *cvq = &mvdev->cvq; 2039 2040 cvq->ready = ready; 2041 if (!ready) 2042 return; 2043 2044 cvq->vring.notify = mlx5_cvq_notify; 2045 } 2046 2047 static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 2048 { 2049 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2050 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2051 struct mlx5_vdpa_virtqueue *mvq; 2052 int err; 2053 2054 if (!mvdev->actual_features) 2055 return; 2056 2057 if (!is_index_valid(mvdev, idx)) 2058 return; 2059 2060 if (is_ctrl_vq_idx(mvdev, idx)) { 2061 set_cvq_ready(mvdev, ready); 2062 return; 2063 } 2064 2065 mvq = &ndev->vqs[idx]; 2066 if (!ready) { 2067 suspend_vq(ndev, mvq); 2068 } else { 2069 err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 2070 if (err) { 2071 mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err); 2072 ready = false; 2073 } 2074 } 2075 2076 2077 mvq->ready = ready; 2078 } 2079 2080 static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx) 2081 { 2082 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2083 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2084 2085 if (!is_index_valid(mvdev, idx)) 2086 return false; 2087 2088 if (is_ctrl_vq_idx(mvdev, idx)) 2089 return mvdev->cvq.ready; 2090 2091 return ndev->vqs[idx].ready; 2092 } 2093 2094 static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx, 2095 const struct vdpa_vq_state *state) 2096 { 2097 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2098 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2099 struct mlx5_vdpa_virtqueue *mvq; 2100 2101 if (!is_index_valid(mvdev, idx)) 2102 return -EINVAL; 2103 2104 if (is_ctrl_vq_idx(mvdev, idx)) { 2105 mvdev->cvq.vring.last_avail_idx = state->split.avail_index; 2106 return 0; 2107 } 2108 2109 mvq = &ndev->vqs[idx]; 2110 if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) { 2111 mlx5_vdpa_warn(mvdev, "can't modify available index\n"); 2112 return -EINVAL; 2113 } 2114 2115 mvq->used_idx = state->split.avail_index; 2116 mvq->avail_idx = state->split.avail_index; 2117 return 0; 2118 } 2119 2120 static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 2121 { 2122 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2123 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2124 struct mlx5_vdpa_virtqueue *mvq; 2125 struct mlx5_virtq_attr attr; 2126 int err; 2127 2128 if (!is_index_valid(mvdev, idx)) 2129 return -EINVAL; 2130 2131 if (is_ctrl_vq_idx(mvdev, idx)) { 2132 state->split.avail_index = mvdev->cvq.vring.last_avail_idx; 2133 return 0; 2134 } 2135 2136 mvq = &ndev->vqs[idx]; 2137 /* If the virtq object was destroyed, use the value saved at 2138 * the last minute of suspend_vq. This caters for userspace 2139 * that cares about emulating the index after vq is stopped. 2140 */ 2141 if (!mvq->initialized) { 2142 /* Firmware returns a wrong value for the available index. 2143 * Since both values should be identical, we take the value of 2144 * used_idx which is reported correctly. 2145 */ 2146 state->split.avail_index = mvq->used_idx; 2147 return 0; 2148 } 2149 2150 err = query_virtqueue(ndev, mvq, &attr); 2151 if (err) { 2152 mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n"); 2153 return err; 2154 } 2155 state->split.avail_index = attr.used_index; 2156 return 0; 2157 } 2158 2159 static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) 2160 { 2161 return PAGE_SIZE; 2162 } 2163 2164 static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx) 2165 { 2166 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2167 2168 if (is_ctrl_vq_idx(mvdev, idx)) 2169 return MLX5_VDPA_CVQ_GROUP; 2170 2171 return MLX5_VDPA_DATAVQ_GROUP; 2172 } 2173 2174 enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9, 2175 MLX5_VIRTIO_NET_F_CSUM = 1 << 10, 2176 MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11, 2177 MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12, 2178 }; 2179 2180 static u64 mlx_to_vritio_features(u16 dev_features) 2181 { 2182 u64 result = 0; 2183 2184 if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM) 2185 result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM); 2186 if (dev_features & MLX5_VIRTIO_NET_F_CSUM) 2187 result |= BIT_ULL(VIRTIO_NET_F_CSUM); 2188 if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6) 2189 result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6); 2190 if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4) 2191 result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4); 2192 2193 return result; 2194 } 2195 2196 static u64 get_supported_features(struct mlx5_core_dev *mdev) 2197 { 2198 u64 mlx_vdpa_features = 0; 2199 u16 dev_features; 2200 2201 dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mdev, device_features_bits_mask); 2202 mlx_vdpa_features |= mlx_to_vritio_features(dev_features); 2203 if (MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_version_1_0)) 2204 mlx_vdpa_features |= BIT_ULL(VIRTIO_F_VERSION_1); 2205 mlx_vdpa_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM); 2206 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VQ); 2207 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR); 2208 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MQ); 2209 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_STATUS); 2210 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MTU); 2211 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VLAN); 2212 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MAC); 2213 2214 return mlx_vdpa_features; 2215 } 2216 2217 static u64 mlx5_vdpa_get_device_features(struct vdpa_device *vdev) 2218 { 2219 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2220 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2221 2222 print_features(mvdev, ndev->mvdev.mlx_features, false); 2223 return ndev->mvdev.mlx_features; 2224 } 2225 2226 static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features) 2227 { 2228 /* Minimum features to expect */ 2229 if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) 2230 return -EOPNOTSUPP; 2231 2232 /* Double check features combination sent down by the driver. 2233 * Fail invalid features due to absence of the depended feature. 2234 * 2235 * Per VIRTIO v1.1 specification, section 5.1.3.1 Feature bit 2236 * requirements: "VIRTIO_NET_F_MQ Requires VIRTIO_NET_F_CTRL_VQ". 2237 * By failing the invalid features sent down by untrusted drivers, 2238 * we're assured the assumption made upon is_index_valid() and 2239 * is_ctrl_vq_idx() will not be compromised. 2240 */ 2241 if ((features & (BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) == 2242 BIT_ULL(VIRTIO_NET_F_MQ)) 2243 return -EINVAL; 2244 2245 return 0; 2246 } 2247 2248 static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev) 2249 { 2250 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2251 int err; 2252 int i; 2253 2254 for (i = 0; i < mvdev->max_vqs; i++) { 2255 err = setup_vq(ndev, &ndev->vqs[i]); 2256 if (err) 2257 goto err_vq; 2258 } 2259 2260 return 0; 2261 2262 err_vq: 2263 for (--i; i >= 0; i--) 2264 teardown_vq(ndev, &ndev->vqs[i]); 2265 2266 return err; 2267 } 2268 2269 static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) 2270 { 2271 struct mlx5_vdpa_virtqueue *mvq; 2272 int i; 2273 2274 for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { 2275 mvq = &ndev->vqs[i]; 2276 if (!mvq->initialized) 2277 continue; 2278 2279 teardown_vq(ndev, mvq); 2280 } 2281 } 2282 2283 static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) 2284 { 2285 if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_CTRL_VQ)) { 2286 if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) { 2287 /* MQ supported. CVQ index is right above the last data virtqueue's */ 2288 mvdev->max_idx = mvdev->max_vqs; 2289 } else { 2290 /* Only CVQ supportted. data virtqueues occupy indices 0 and 1. 2291 * CVQ gets index 2 2292 */ 2293 mvdev->max_idx = 2; 2294 } 2295 } else { 2296 /* Two data virtqueues only: one for rx and one for tx */ 2297 mvdev->max_idx = 1; 2298 } 2299 } 2300 2301 static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) 2302 { 2303 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2304 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2305 int err; 2306 2307 print_features(mvdev, features, true); 2308 2309 err = verify_driver_features(mvdev, features); 2310 if (err) 2311 return err; 2312 2313 ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; 2314 if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)) 2315 ndev->rqt_size = mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs); 2316 else 2317 ndev->rqt_size = 1; 2318 2319 ndev->cur_num_vqs = 2 * ndev->rqt_size; 2320 2321 update_cvq_info(mvdev); 2322 return err; 2323 } 2324 2325 static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 2326 { 2327 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2328 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2329 2330 ndev->config_cb = *cb; 2331 } 2332 2333 #define MLX5_VDPA_MAX_VQ_ENTRIES 256 2334 static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev) 2335 { 2336 return MLX5_VDPA_MAX_VQ_ENTRIES; 2337 } 2338 2339 static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev) 2340 { 2341 return VIRTIO_ID_NET; 2342 } 2343 2344 static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev) 2345 { 2346 return PCI_VENDOR_ID_MELLANOX; 2347 } 2348 2349 static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev) 2350 { 2351 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2352 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2353 2354 print_status(mvdev, ndev->mvdev.status, false); 2355 return ndev->mvdev.status; 2356 } 2357 2358 static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 2359 { 2360 struct mlx5_vq_restore_info *ri = &mvq->ri; 2361 struct mlx5_virtq_attr attr = {}; 2362 int err; 2363 2364 if (mvq->initialized) { 2365 err = query_virtqueue(ndev, mvq, &attr); 2366 if (err) 2367 return err; 2368 } 2369 2370 ri->avail_index = attr.available_index; 2371 ri->used_index = attr.used_index; 2372 ri->ready = mvq->ready; 2373 ri->num_ent = mvq->num_ent; 2374 ri->desc_addr = mvq->desc_addr; 2375 ri->device_addr = mvq->device_addr; 2376 ri->driver_addr = mvq->driver_addr; 2377 ri->restore = true; 2378 return 0; 2379 } 2380 2381 static int save_channels_info(struct mlx5_vdpa_net *ndev) 2382 { 2383 int i; 2384 2385 for (i = 0; i < ndev->mvdev.max_vqs; i++) { 2386 memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); 2387 save_channel_info(ndev, &ndev->vqs[i]); 2388 } 2389 return 0; 2390 } 2391 2392 static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) 2393 { 2394 int i; 2395 2396 for (i = 0; i < ndev->mvdev.max_vqs; i++) 2397 memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 2398 } 2399 2400 static void restore_channels_info(struct mlx5_vdpa_net *ndev) 2401 { 2402 struct mlx5_vdpa_virtqueue *mvq; 2403 struct mlx5_vq_restore_info *ri; 2404 int i; 2405 2406 mlx5_clear_vqs(ndev); 2407 init_mvqs(ndev); 2408 for (i = 0; i < ndev->mvdev.max_vqs; i++) { 2409 mvq = &ndev->vqs[i]; 2410 ri = &mvq->ri; 2411 if (!ri->restore) 2412 continue; 2413 2414 mvq->avail_idx = ri->avail_index; 2415 mvq->used_idx = ri->used_index; 2416 mvq->ready = ri->ready; 2417 mvq->num_ent = ri->num_ent; 2418 mvq->desc_addr = ri->desc_addr; 2419 mvq->device_addr = ri->device_addr; 2420 mvq->driver_addr = ri->driver_addr; 2421 } 2422 } 2423 2424 static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, 2425 struct vhost_iotlb *iotlb, unsigned int asid) 2426 { 2427 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2428 int err; 2429 2430 suspend_vqs(ndev); 2431 err = save_channels_info(ndev); 2432 if (err) 2433 goto err_mr; 2434 2435 teardown_driver(ndev); 2436 mlx5_vdpa_destroy_mr(mvdev); 2437 err = mlx5_vdpa_create_mr(mvdev, iotlb, asid); 2438 if (err) 2439 goto err_mr; 2440 2441 if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK) || mvdev->suspended) 2442 goto err_mr; 2443 2444 restore_channels_info(ndev); 2445 err = setup_driver(mvdev); 2446 if (err) 2447 goto err_setup; 2448 2449 return 0; 2450 2451 err_setup: 2452 mlx5_vdpa_destroy_mr(mvdev); 2453 err_mr: 2454 return err; 2455 } 2456 2457 /* reslock must be held for this function */ 2458 static int setup_driver(struct mlx5_vdpa_dev *mvdev) 2459 { 2460 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2461 int err; 2462 2463 WARN_ON(!rwsem_is_locked(&ndev->reslock)); 2464 2465 if (ndev->setup) { 2466 mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n"); 2467 err = 0; 2468 goto out; 2469 } 2470 err = setup_virtqueues(mvdev); 2471 if (err) { 2472 mlx5_vdpa_warn(mvdev, "setup_virtqueues\n"); 2473 goto out; 2474 } 2475 2476 err = create_rqt(ndev); 2477 if (err) { 2478 mlx5_vdpa_warn(mvdev, "create_rqt\n"); 2479 goto err_rqt; 2480 } 2481 2482 err = create_tir(ndev); 2483 if (err) { 2484 mlx5_vdpa_warn(mvdev, "create_tir\n"); 2485 goto err_tir; 2486 } 2487 2488 err = setup_steering(ndev); 2489 if (err) { 2490 mlx5_vdpa_warn(mvdev, "setup_steering\n"); 2491 goto err_fwd; 2492 } 2493 ndev->setup = true; 2494 2495 return 0; 2496 2497 err_fwd: 2498 destroy_tir(ndev); 2499 err_tir: 2500 destroy_rqt(ndev); 2501 err_rqt: 2502 teardown_virtqueues(ndev); 2503 out: 2504 return err; 2505 } 2506 2507 /* reslock must be held for this function */ 2508 static void teardown_driver(struct mlx5_vdpa_net *ndev) 2509 { 2510 2511 WARN_ON(!rwsem_is_locked(&ndev->reslock)); 2512 2513 if (!ndev->setup) 2514 return; 2515 2516 teardown_steering(ndev); 2517 destroy_tir(ndev); 2518 destroy_rqt(ndev); 2519 teardown_virtqueues(ndev); 2520 ndev->setup = false; 2521 } 2522 2523 static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) 2524 { 2525 int i; 2526 2527 for (i = 0; i < ndev->mvdev.max_vqs; i++) 2528 ndev->vqs[i].ready = false; 2529 2530 ndev->mvdev.cvq.ready = false; 2531 } 2532 2533 static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev) 2534 { 2535 struct mlx5_control_vq *cvq = &mvdev->cvq; 2536 int err = 0; 2537 2538 if (mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) 2539 err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features, 2540 MLX5_CVQ_MAX_ENT, false, 2541 (struct vring_desc *)(uintptr_t)cvq->desc_addr, 2542 (struct vring_avail *)(uintptr_t)cvq->driver_addr, 2543 (struct vring_used *)(uintptr_t)cvq->device_addr); 2544 2545 return err; 2546 } 2547 2548 static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) 2549 { 2550 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2551 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2552 int err; 2553 2554 print_status(mvdev, status, true); 2555 2556 down_write(&ndev->reslock); 2557 2558 if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { 2559 if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 2560 err = setup_cvq_vring(mvdev); 2561 if (err) { 2562 mlx5_vdpa_warn(mvdev, "failed to setup control VQ vring\n"); 2563 goto err_setup; 2564 } 2565 err = setup_driver(mvdev); 2566 if (err) { 2567 mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); 2568 goto err_setup; 2569 } 2570 } else { 2571 mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); 2572 goto err_clear; 2573 } 2574 } 2575 2576 ndev->mvdev.status = status; 2577 up_write(&ndev->reslock); 2578 return; 2579 2580 err_setup: 2581 mlx5_vdpa_destroy_mr(&ndev->mvdev); 2582 ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; 2583 err_clear: 2584 up_write(&ndev->reslock); 2585 } 2586 2587 static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev) 2588 { 2589 int i; 2590 2591 /* default mapping all groups are mapped to asid 0 */ 2592 for (i = 0; i < MLX5_VDPA_NUMVQ_GROUPS; i++) 2593 mvdev->group2asid[i] = 0; 2594 } 2595 2596 static int mlx5_vdpa_reset(struct vdpa_device *vdev) 2597 { 2598 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2599 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2600 2601 print_status(mvdev, 0, true); 2602 mlx5_vdpa_info(mvdev, "performing device reset\n"); 2603 2604 down_write(&ndev->reslock); 2605 teardown_driver(ndev); 2606 clear_vqs_ready(ndev); 2607 mlx5_vdpa_destroy_mr(&ndev->mvdev); 2608 ndev->mvdev.status = 0; 2609 ndev->mvdev.suspended = false; 2610 ndev->cur_num_vqs = 0; 2611 ndev->mvdev.cvq.received_desc = 0; 2612 ndev->mvdev.cvq.completed_desc = 0; 2613 memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1)); 2614 ndev->mvdev.actual_features = 0; 2615 init_group_to_asid_map(mvdev); 2616 ++mvdev->generation; 2617 2618 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 2619 if (mlx5_vdpa_create_mr(mvdev, NULL, 0)) 2620 mlx5_vdpa_warn(mvdev, "create MR failed\n"); 2621 } 2622 up_write(&ndev->reslock); 2623 2624 return 0; 2625 } 2626 2627 static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev) 2628 { 2629 return sizeof(struct virtio_net_config); 2630 } 2631 2632 static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, 2633 unsigned int len) 2634 { 2635 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2636 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2637 2638 if (offset + len <= sizeof(struct virtio_net_config)) 2639 memcpy(buf, (u8 *)&ndev->config + offset, len); 2640 } 2641 2642 static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf, 2643 unsigned int len) 2644 { 2645 /* not supported */ 2646 } 2647 2648 static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) 2649 { 2650 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2651 2652 return mvdev->generation; 2653 } 2654 2655 static int set_map_data(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb, 2656 unsigned int asid) 2657 { 2658 bool change_map; 2659 int err; 2660 2661 err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map, asid); 2662 if (err) { 2663 mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); 2664 return err; 2665 } 2666 2667 if (change_map) 2668 err = mlx5_vdpa_change_map(mvdev, iotlb, asid); 2669 2670 return err; 2671 } 2672 2673 static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid, 2674 struct vhost_iotlb *iotlb) 2675 { 2676 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2677 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2678 int err = -EINVAL; 2679 2680 down_write(&ndev->reslock); 2681 err = set_map_data(mvdev, iotlb, asid); 2682 up_write(&ndev->reslock); 2683 return err; 2684 } 2685 2686 static struct device *mlx5_get_vq_dma_dev(struct vdpa_device *vdev, u16 idx) 2687 { 2688 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2689 2690 if (is_ctrl_vq_idx(mvdev, idx)) 2691 return &vdev->dev; 2692 2693 return mvdev->vdev.dma_dev; 2694 } 2695 2696 static void mlx5_vdpa_free(struct vdpa_device *vdev) 2697 { 2698 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2699 struct mlx5_core_dev *pfmdev; 2700 struct mlx5_vdpa_net *ndev; 2701 2702 ndev = to_mlx5_vdpa_ndev(mvdev); 2703 2704 free_resources(ndev); 2705 mlx5_vdpa_destroy_mr(mvdev); 2706 if (!is_zero_ether_addr(ndev->config.mac)) { 2707 pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 2708 mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); 2709 } 2710 mlx5_vdpa_free_resources(&ndev->mvdev); 2711 kfree(ndev->event_cbs); 2712 kfree(ndev->vqs); 2713 } 2714 2715 static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx) 2716 { 2717 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2718 struct vdpa_notification_area ret = {}; 2719 struct mlx5_vdpa_net *ndev; 2720 phys_addr_t addr; 2721 2722 if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2723 return ret; 2724 2725 /* If SF BAR size is smaller than PAGE_SIZE, do not use direct 2726 * notification to avoid the risk of mapping pages that contain BAR of more 2727 * than one SF 2728 */ 2729 if (MLX5_CAP_GEN(mvdev->mdev, log_min_sf_size) + 12 < PAGE_SHIFT) 2730 return ret; 2731 2732 ndev = to_mlx5_vdpa_ndev(mvdev); 2733 addr = (phys_addr_t)ndev->mvdev.res.phys_kick_addr; 2734 ret.addr = addr; 2735 ret.size = PAGE_SIZE; 2736 return ret; 2737 } 2738 2739 static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) 2740 { 2741 return -EOPNOTSUPP; 2742 } 2743 2744 static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) 2745 { 2746 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2747 2748 return mvdev->actual_features; 2749 } 2750 2751 static int counter_set_query(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 2752 u64 *received_desc, u64 *completed_desc) 2753 { 2754 u32 in[MLX5_ST_SZ_DW(query_virtio_q_counters_in)] = {}; 2755 u32 out[MLX5_ST_SZ_DW(query_virtio_q_counters_out)] = {}; 2756 void *cmd_hdr; 2757 void *ctx; 2758 int err; 2759 2760 if (!counters_supported(&ndev->mvdev)) 2761 return -EOPNOTSUPP; 2762 2763 if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 2764 return -EAGAIN; 2765 2766 cmd_hdr = MLX5_ADDR_OF(query_virtio_q_counters_in, in, hdr); 2767 2768 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 2769 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 2770 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 2771 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->counter_set_id); 2772 2773 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 2774 if (err) 2775 return err; 2776 2777 ctx = MLX5_ADDR_OF(query_virtio_q_counters_out, out, counters); 2778 *received_desc = MLX5_GET64(virtio_q_counters, ctx, received_desc); 2779 *completed_desc = MLX5_GET64(virtio_q_counters, ctx, completed_desc); 2780 return 0; 2781 } 2782 2783 static int mlx5_vdpa_get_vendor_vq_stats(struct vdpa_device *vdev, u16 idx, 2784 struct sk_buff *msg, 2785 struct netlink_ext_ack *extack) 2786 { 2787 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2788 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2789 struct mlx5_vdpa_virtqueue *mvq; 2790 struct mlx5_control_vq *cvq; 2791 u64 received_desc; 2792 u64 completed_desc; 2793 int err = 0; 2794 2795 down_read(&ndev->reslock); 2796 if (!is_index_valid(mvdev, idx)) { 2797 NL_SET_ERR_MSG_MOD(extack, "virtqueue index is not valid"); 2798 err = -EINVAL; 2799 goto out_err; 2800 } 2801 2802 if (idx == ctrl_vq_idx(mvdev)) { 2803 cvq = &mvdev->cvq; 2804 received_desc = cvq->received_desc; 2805 completed_desc = cvq->completed_desc; 2806 goto out; 2807 } 2808 2809 mvq = &ndev->vqs[idx]; 2810 err = counter_set_query(ndev, mvq, &received_desc, &completed_desc); 2811 if (err) { 2812 NL_SET_ERR_MSG_MOD(extack, "failed to query hardware"); 2813 goto out_err; 2814 } 2815 2816 out: 2817 err = -EMSGSIZE; 2818 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "received_desc")) 2819 goto out_err; 2820 2821 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, received_desc, 2822 VDPA_ATTR_PAD)) 2823 goto out_err; 2824 2825 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "completed_desc")) 2826 goto out_err; 2827 2828 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, completed_desc, 2829 VDPA_ATTR_PAD)) 2830 goto out_err; 2831 2832 err = 0; 2833 out_err: 2834 up_read(&ndev->reslock); 2835 return err; 2836 } 2837 2838 static void mlx5_vdpa_cvq_suspend(struct mlx5_vdpa_dev *mvdev) 2839 { 2840 struct mlx5_control_vq *cvq; 2841 2842 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 2843 return; 2844 2845 cvq = &mvdev->cvq; 2846 cvq->ready = false; 2847 } 2848 2849 static int mlx5_vdpa_suspend(struct vdpa_device *vdev) 2850 { 2851 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2852 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2853 struct mlx5_vdpa_virtqueue *mvq; 2854 int i; 2855 2856 mlx5_vdpa_info(mvdev, "suspending device\n"); 2857 2858 down_write(&ndev->reslock); 2859 ndev->nb_registered = false; 2860 mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 2861 flush_workqueue(ndev->mvdev.wq); 2862 for (i = 0; i < ndev->cur_num_vqs; i++) { 2863 mvq = &ndev->vqs[i]; 2864 suspend_vq(ndev, mvq); 2865 } 2866 mlx5_vdpa_cvq_suspend(mvdev); 2867 mvdev->suspended = true; 2868 up_write(&ndev->reslock); 2869 return 0; 2870 } 2871 2872 static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group, 2873 unsigned int asid) 2874 { 2875 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2876 2877 if (group >= MLX5_VDPA_NUMVQ_GROUPS) 2878 return -EINVAL; 2879 2880 mvdev->group2asid[group] = asid; 2881 return 0; 2882 } 2883 2884 static const struct vdpa_config_ops mlx5_vdpa_ops = { 2885 .set_vq_address = mlx5_vdpa_set_vq_address, 2886 .set_vq_num = mlx5_vdpa_set_vq_num, 2887 .kick_vq = mlx5_vdpa_kick_vq, 2888 .set_vq_cb = mlx5_vdpa_set_vq_cb, 2889 .set_vq_ready = mlx5_vdpa_set_vq_ready, 2890 .get_vq_ready = mlx5_vdpa_get_vq_ready, 2891 .set_vq_state = mlx5_vdpa_set_vq_state, 2892 .get_vq_state = mlx5_vdpa_get_vq_state, 2893 .get_vendor_vq_stats = mlx5_vdpa_get_vendor_vq_stats, 2894 .get_vq_notification = mlx5_get_vq_notification, 2895 .get_vq_irq = mlx5_get_vq_irq, 2896 .get_vq_align = mlx5_vdpa_get_vq_align, 2897 .get_vq_group = mlx5_vdpa_get_vq_group, 2898 .get_device_features = mlx5_vdpa_get_device_features, 2899 .set_driver_features = mlx5_vdpa_set_driver_features, 2900 .get_driver_features = mlx5_vdpa_get_driver_features, 2901 .set_config_cb = mlx5_vdpa_set_config_cb, 2902 .get_vq_num_max = mlx5_vdpa_get_vq_num_max, 2903 .get_device_id = mlx5_vdpa_get_device_id, 2904 .get_vendor_id = mlx5_vdpa_get_vendor_id, 2905 .get_status = mlx5_vdpa_get_status, 2906 .set_status = mlx5_vdpa_set_status, 2907 .reset = mlx5_vdpa_reset, 2908 .get_config_size = mlx5_vdpa_get_config_size, 2909 .get_config = mlx5_vdpa_get_config, 2910 .set_config = mlx5_vdpa_set_config, 2911 .get_generation = mlx5_vdpa_get_generation, 2912 .set_map = mlx5_vdpa_set_map, 2913 .set_group_asid = mlx5_set_group_asid, 2914 .get_vq_dma_dev = mlx5_get_vq_dma_dev, 2915 .free = mlx5_vdpa_free, 2916 .suspend = mlx5_vdpa_suspend, 2917 }; 2918 2919 static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) 2920 { 2921 u16 hw_mtu; 2922 int err; 2923 2924 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2925 if (err) 2926 return err; 2927 2928 *mtu = hw_mtu - MLX5V_ETH_HARD_MTU; 2929 return 0; 2930 } 2931 2932 static int alloc_resources(struct mlx5_vdpa_net *ndev) 2933 { 2934 struct mlx5_vdpa_net_resources *res = &ndev->res; 2935 int err; 2936 2937 if (res->valid) { 2938 mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); 2939 return -EEXIST; 2940 } 2941 2942 err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); 2943 if (err) 2944 return err; 2945 2946 err = create_tis(ndev); 2947 if (err) 2948 goto err_tis; 2949 2950 res->valid = true; 2951 2952 return 0; 2953 2954 err_tis: 2955 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 2956 return err; 2957 } 2958 2959 static void free_resources(struct mlx5_vdpa_net *ndev) 2960 { 2961 struct mlx5_vdpa_net_resources *res = &ndev->res; 2962 2963 if (!res->valid) 2964 return; 2965 2966 destroy_tis(ndev); 2967 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 2968 res->valid = false; 2969 } 2970 2971 static void init_mvqs(struct mlx5_vdpa_net *ndev) 2972 { 2973 struct mlx5_vdpa_virtqueue *mvq; 2974 int i; 2975 2976 for (i = 0; i < ndev->mvdev.max_vqs; ++i) { 2977 mvq = &ndev->vqs[i]; 2978 memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 2979 mvq->index = i; 2980 mvq->ndev = ndev; 2981 mvq->fwqp.fw = true; 2982 mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; 2983 } 2984 for (; i < ndev->mvdev.max_vqs; i++) { 2985 mvq = &ndev->vqs[i]; 2986 memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 2987 mvq->index = i; 2988 mvq->ndev = ndev; 2989 } 2990 } 2991 2992 struct mlx5_vdpa_mgmtdev { 2993 struct vdpa_mgmt_dev mgtdev; 2994 struct mlx5_adev *madev; 2995 struct mlx5_vdpa_net *ndev; 2996 }; 2997 2998 static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) 2999 { 3000 u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; 3001 u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; 3002 int err; 3003 3004 MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE); 3005 MLX5_SET(query_vport_state_in, in, op_mod, opmod); 3006 MLX5_SET(query_vport_state_in, in, vport_number, vport); 3007 if (vport) 3008 MLX5_SET(query_vport_state_in, in, other_vport, 1); 3009 3010 err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out); 3011 if (err) 3012 return 0; 3013 3014 return MLX5_GET(query_vport_state_out, out, state); 3015 } 3016 3017 static bool get_link_state(struct mlx5_vdpa_dev *mvdev) 3018 { 3019 if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) == 3020 VPORT_STATE_UP) 3021 return true; 3022 3023 return false; 3024 } 3025 3026 static void update_carrier(struct work_struct *work) 3027 { 3028 struct mlx5_vdpa_wq_ent *wqent; 3029 struct mlx5_vdpa_dev *mvdev; 3030 struct mlx5_vdpa_net *ndev; 3031 3032 wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 3033 mvdev = wqent->mvdev; 3034 ndev = to_mlx5_vdpa_ndev(mvdev); 3035 if (get_link_state(mvdev)) 3036 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 3037 else 3038 ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 3039 3040 if (ndev->nb_registered && ndev->config_cb.callback) 3041 ndev->config_cb.callback(ndev->config_cb.private); 3042 3043 kfree(wqent); 3044 } 3045 3046 static int event_handler(struct notifier_block *nb, unsigned long event, void *param) 3047 { 3048 struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb); 3049 struct mlx5_eqe *eqe = param; 3050 int ret = NOTIFY_DONE; 3051 struct mlx5_vdpa_wq_ent *wqent; 3052 3053 if (event == MLX5_EVENT_TYPE_PORT_CHANGE) { 3054 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_STATUS))) 3055 return NOTIFY_DONE; 3056 switch (eqe->sub_type) { 3057 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 3058 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 3059 wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); 3060 if (!wqent) 3061 return NOTIFY_DONE; 3062 3063 wqent->mvdev = &ndev->mvdev; 3064 INIT_WORK(&wqent->work, update_carrier); 3065 queue_work(ndev->mvdev.wq, &wqent->work); 3066 ret = NOTIFY_OK; 3067 break; 3068 default: 3069 return NOTIFY_DONE; 3070 } 3071 return ret; 3072 } 3073 return ret; 3074 } 3075 3076 static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu) 3077 { 3078 int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); 3079 void *in; 3080 int err; 3081 3082 in = kvzalloc(inlen, GFP_KERNEL); 3083 if (!in) 3084 return -ENOMEM; 3085 3086 MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1); 3087 MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu, 3088 mtu + MLX5V_ETH_HARD_MTU); 3089 MLX5_SET(modify_nic_vport_context_in, in, opcode, 3090 MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); 3091 3092 err = mlx5_cmd_exec_in(mdev, modify_nic_vport_context, in); 3093 3094 kvfree(in); 3095 return err; 3096 } 3097 3098 static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, 3099 const struct vdpa_dev_set_config *add_config) 3100 { 3101 struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 3102 struct virtio_net_config *config; 3103 struct mlx5_core_dev *pfmdev; 3104 struct mlx5_vdpa_dev *mvdev; 3105 struct mlx5_vdpa_net *ndev; 3106 struct mlx5_core_dev *mdev; 3107 u64 device_features; 3108 u32 max_vqs; 3109 u16 mtu; 3110 int err; 3111 3112 if (mgtdev->ndev) 3113 return -ENOSPC; 3114 3115 mdev = mgtdev->madev->mdev; 3116 device_features = mgtdev->mgtdev.supported_features; 3117 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { 3118 if (add_config->device_features & ~device_features) { 3119 dev_warn(mdev->device, 3120 "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n", 3121 add_config->device_features, device_features); 3122 return -EINVAL; 3123 } 3124 device_features &= add_config->device_features; 3125 } 3126 if (!(device_features & BIT_ULL(VIRTIO_F_VERSION_1) && 3127 device_features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) { 3128 dev_warn(mdev->device, 3129 "Must provision minimum features 0x%llx for this device", 3130 BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)); 3131 return -EOPNOTSUPP; 3132 } 3133 3134 if (!(MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_queue_type) & 3135 MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)) { 3136 dev_warn(mdev->device, "missing support for split virtqueues\n"); 3137 return -EOPNOTSUPP; 3138 } 3139 3140 max_vqs = min_t(int, MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues), 3141 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size)); 3142 if (max_vqs < 2) { 3143 dev_warn(mdev->device, 3144 "%d virtqueues are supported. At least 2 are required\n", 3145 max_vqs); 3146 return -EAGAIN; 3147 } 3148 3149 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP)) { 3150 if (add_config->net.max_vq_pairs > max_vqs / 2) 3151 return -EINVAL; 3152 max_vqs = min_t(u32, max_vqs, 2 * add_config->net.max_vq_pairs); 3153 } else { 3154 max_vqs = 2; 3155 } 3156 3157 ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, 3158 MLX5_VDPA_NUMVQ_GROUPS, MLX5_VDPA_NUM_AS, name, false); 3159 if (IS_ERR(ndev)) 3160 return PTR_ERR(ndev); 3161 3162 ndev->mvdev.max_vqs = max_vqs; 3163 mvdev = &ndev->mvdev; 3164 mvdev->mdev = mdev; 3165 3166 ndev->vqs = kcalloc(max_vqs, sizeof(*ndev->vqs), GFP_KERNEL); 3167 ndev->event_cbs = kcalloc(max_vqs + 1, sizeof(*ndev->event_cbs), GFP_KERNEL); 3168 if (!ndev->vqs || !ndev->event_cbs) { 3169 err = -ENOMEM; 3170 goto err_alloc; 3171 } 3172 3173 init_mvqs(ndev); 3174 init_rwsem(&ndev->reslock); 3175 config = &ndev->config; 3176 3177 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU)) { 3178 err = config_func_mtu(mdev, add_config->net.mtu); 3179 if (err) 3180 goto err_alloc; 3181 } 3182 3183 if (device_features & BIT_ULL(VIRTIO_NET_F_MTU)) { 3184 err = query_mtu(mdev, &mtu); 3185 if (err) 3186 goto err_alloc; 3187 3188 ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, mtu); 3189 } 3190 3191 if (device_features & BIT_ULL(VIRTIO_NET_F_STATUS)) { 3192 if (get_link_state(mvdev)) 3193 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 3194 else 3195 ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 3196 } 3197 3198 if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) { 3199 memcpy(ndev->config.mac, add_config->net.mac, ETH_ALEN); 3200 /* No bother setting mac address in config if not going to provision _F_MAC */ 3201 } else if ((add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) == 0 || 3202 device_features & BIT_ULL(VIRTIO_NET_F_MAC)) { 3203 err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac); 3204 if (err) 3205 goto err_alloc; 3206 } 3207 3208 if (!is_zero_ether_addr(config->mac)) { 3209 pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev)); 3210 err = mlx5_mpfs_add_mac(pfmdev, config->mac); 3211 if (err) 3212 goto err_alloc; 3213 } else if ((add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) == 0) { 3214 /* 3215 * We used to clear _F_MAC feature bit if seeing 3216 * zero mac address when device features are not 3217 * specifically provisioned. Keep the behaviour 3218 * so old scripts do not break. 3219 */ 3220 device_features &= ~BIT_ULL(VIRTIO_NET_F_MAC); 3221 } else if (device_features & BIT_ULL(VIRTIO_NET_F_MAC)) { 3222 /* Don't provision zero mac address for _F_MAC */ 3223 mlx5_vdpa_warn(&ndev->mvdev, 3224 "No mac address provisioned?\n"); 3225 err = -EINVAL; 3226 goto err_alloc; 3227 } 3228 3229 if (device_features & BIT_ULL(VIRTIO_NET_F_MQ)) 3230 config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, max_vqs / 2); 3231 3232 ndev->mvdev.mlx_features = device_features; 3233 mvdev->vdev.dma_dev = &mdev->pdev->dev; 3234 err = mlx5_vdpa_alloc_resources(&ndev->mvdev); 3235 if (err) 3236 goto err_mpfs; 3237 3238 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 3239 err = mlx5_vdpa_create_mr(mvdev, NULL, 0); 3240 if (err) 3241 goto err_res; 3242 } 3243 3244 err = alloc_resources(ndev); 3245 if (err) 3246 goto err_mr; 3247 3248 ndev->cvq_ent.mvdev = mvdev; 3249 INIT_WORK(&ndev->cvq_ent.work, mlx5_cvq_kick_handler); 3250 mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_wq"); 3251 if (!mvdev->wq) { 3252 err = -ENOMEM; 3253 goto err_res2; 3254 } 3255 3256 ndev->nb.notifier_call = event_handler; 3257 mlx5_notifier_register(mdev, &ndev->nb); 3258 ndev->nb_registered = true; 3259 mvdev->vdev.mdev = &mgtdev->mgtdev; 3260 err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1); 3261 if (err) 3262 goto err_reg; 3263 3264 mlx5_vdpa_add_debugfs(ndev); 3265 mgtdev->ndev = ndev; 3266 return 0; 3267 3268 err_reg: 3269 destroy_workqueue(mvdev->wq); 3270 err_res2: 3271 free_resources(ndev); 3272 err_mr: 3273 mlx5_vdpa_destroy_mr(mvdev); 3274 err_res: 3275 mlx5_vdpa_free_resources(&ndev->mvdev); 3276 err_mpfs: 3277 if (!is_zero_ether_addr(config->mac)) 3278 mlx5_mpfs_del_mac(pfmdev, config->mac); 3279 err_alloc: 3280 put_device(&mvdev->vdev.dev); 3281 return err; 3282 } 3283 3284 static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev) 3285 { 3286 struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 3287 struct mlx5_vdpa_dev *mvdev = to_mvdev(dev); 3288 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 3289 struct workqueue_struct *wq; 3290 3291 mlx5_vdpa_remove_debugfs(ndev->debugfs); 3292 ndev->debugfs = NULL; 3293 if (ndev->nb_registered) { 3294 ndev->nb_registered = false; 3295 mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 3296 } 3297 wq = mvdev->wq; 3298 mvdev->wq = NULL; 3299 destroy_workqueue(wq); 3300 _vdpa_unregister_device(dev); 3301 mgtdev->ndev = NULL; 3302 } 3303 3304 static const struct vdpa_mgmtdev_ops mdev_ops = { 3305 .dev_add = mlx5_vdpa_dev_add, 3306 .dev_del = mlx5_vdpa_dev_del, 3307 }; 3308 3309 static struct virtio_device_id id_table[] = { 3310 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 3311 { 0 }, 3312 }; 3313 3314 static int mlx5v_probe(struct auxiliary_device *adev, 3315 const struct auxiliary_device_id *id) 3316 3317 { 3318 struct mlx5_adev *madev = container_of(adev, struct mlx5_adev, adev); 3319 struct mlx5_core_dev *mdev = madev->mdev; 3320 struct mlx5_vdpa_mgmtdev *mgtdev; 3321 int err; 3322 3323 mgtdev = kzalloc(sizeof(*mgtdev), GFP_KERNEL); 3324 if (!mgtdev) 3325 return -ENOMEM; 3326 3327 mgtdev->mgtdev.ops = &mdev_ops; 3328 mgtdev->mgtdev.device = mdev->device; 3329 mgtdev->mgtdev.id_table = id_table; 3330 mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | 3331 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP) | 3332 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU) | 3333 BIT_ULL(VDPA_ATTR_DEV_FEATURES); 3334 mgtdev->mgtdev.max_supported_vqs = 3335 MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1; 3336 mgtdev->mgtdev.supported_features = get_supported_features(mdev); 3337 mgtdev->madev = madev; 3338 3339 err = vdpa_mgmtdev_register(&mgtdev->mgtdev); 3340 if (err) 3341 goto reg_err; 3342 3343 auxiliary_set_drvdata(adev, mgtdev); 3344 3345 return 0; 3346 3347 reg_err: 3348 kfree(mgtdev); 3349 return err; 3350 } 3351 3352 static void mlx5v_remove(struct auxiliary_device *adev) 3353 { 3354 struct mlx5_vdpa_mgmtdev *mgtdev; 3355 3356 mgtdev = auxiliary_get_drvdata(adev); 3357 vdpa_mgmtdev_unregister(&mgtdev->mgtdev); 3358 kfree(mgtdev); 3359 } 3360 3361 static const struct auxiliary_device_id mlx5v_id_table[] = { 3362 { .name = MLX5_ADEV_NAME ".vnet", }, 3363 {}, 3364 }; 3365 3366 MODULE_DEVICE_TABLE(auxiliary, mlx5v_id_table); 3367 3368 static struct auxiliary_driver mlx5v_driver = { 3369 .name = "vnet", 3370 .probe = mlx5v_probe, 3371 .remove = mlx5v_remove, 3372 .id_table = mlx5v_id_table, 3373 }; 3374 3375 module_auxiliary_driver(mlx5v_driver); 3376