1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 3 4 #include <linux/module.h> 5 #include <linux/vdpa.h> 6 #include <linux/vringh.h> 7 #include <uapi/linux/virtio_net.h> 8 #include <uapi/linux/virtio_ids.h> 9 #include <uapi/linux/vdpa.h> 10 #include <linux/virtio_config.h> 11 #include <linux/auxiliary_bus.h> 12 #include <linux/mlx5/cq.h> 13 #include <linux/mlx5/qp.h> 14 #include <linux/mlx5/device.h> 15 #include <linux/mlx5/driver.h> 16 #include <linux/mlx5/vport.h> 17 #include <linux/mlx5/fs.h> 18 #include <linux/mlx5/mlx5_ifc_vdpa.h> 19 #include <linux/mlx5/mpfs.h> 20 #include "mlx5_vdpa.h" 21 #include "mlx5_vnet.h" 22 23 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 24 MODULE_DESCRIPTION("Mellanox VDPA driver"); 25 MODULE_LICENSE("Dual BSD/GPL"); 26 27 #define VALID_FEATURES_MASK \ 28 (BIT_ULL(VIRTIO_NET_F_CSUM) | BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | \ 29 BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_MAC) | \ 30 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | \ 31 BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | \ 32 BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | BIT_ULL(VIRTIO_NET_F_HOST_ECN) | BIT_ULL(VIRTIO_NET_F_HOST_UFO) | \ 33 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | BIT_ULL(VIRTIO_NET_F_STATUS) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | \ 34 BIT_ULL(VIRTIO_NET_F_CTRL_RX) | BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | \ 35 BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE) | \ 36 BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | \ 37 BIT_ULL(VIRTIO_NET_F_RSS) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY) | \ 38 BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX) | BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY) | \ 39 BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | \ 40 BIT_ULL(VIRTIO_F_RING_PACKED) | BIT_ULL(VIRTIO_F_ORDER_PLATFORM) | BIT_ULL(VIRTIO_F_SR_IOV)) 41 42 #define VALID_STATUS_MASK \ 43 (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \ 44 VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED) 45 46 #define MLX5_FEATURE(_mvdev, _feature) (!!((_mvdev)->actual_features & BIT_ULL(_feature))) 47 48 #define MLX5V_UNTAGGED 0x1000 49 50 struct mlx5_vdpa_cq_buf { 51 struct mlx5_frag_buf_ctrl fbc; 52 struct mlx5_frag_buf frag_buf; 53 int cqe_size; 54 int nent; 55 }; 56 57 struct mlx5_vdpa_cq { 58 struct mlx5_core_cq mcq; 59 struct mlx5_vdpa_cq_buf buf; 60 struct mlx5_db db; 61 int cqe; 62 }; 63 64 struct mlx5_vdpa_umem { 65 struct mlx5_frag_buf_ctrl fbc; 66 struct mlx5_frag_buf frag_buf; 67 int size; 68 u32 id; 69 }; 70 71 struct mlx5_vdpa_qp { 72 struct mlx5_core_qp mqp; 73 struct mlx5_frag_buf frag_buf; 74 struct mlx5_db db; 75 u16 head; 76 bool fw; 77 }; 78 79 struct mlx5_vq_restore_info { 80 u32 num_ent; 81 u64 desc_addr; 82 u64 device_addr; 83 u64 driver_addr; 84 u16 avail_index; 85 u16 used_index; 86 bool ready; 87 bool restore; 88 }; 89 90 struct mlx5_vdpa_virtqueue { 91 bool ready; 92 u64 desc_addr; 93 u64 device_addr; 94 u64 driver_addr; 95 u32 num_ent; 96 97 /* Resources for implementing the notification channel from the device 98 * to the driver. fwqp is the firmware end of an RC connection; the 99 * other end is vqqp used by the driver. cq is where completions are 100 * reported. 101 */ 102 struct mlx5_vdpa_cq cq; 103 struct mlx5_vdpa_qp fwqp; 104 struct mlx5_vdpa_qp vqqp; 105 106 /* umem resources are required for the virtqueue operation. They're use 107 * is internal and they must be provided by the driver. 108 */ 109 struct mlx5_vdpa_umem umem1; 110 struct mlx5_vdpa_umem umem2; 111 struct mlx5_vdpa_umem umem3; 112 113 u32 counter_set_id; 114 bool initialized; 115 int index; 116 u32 virtq_id; 117 struct mlx5_vdpa_net *ndev; 118 u16 avail_idx; 119 u16 used_idx; 120 int fw_state; 121 122 /* keep last in the struct */ 123 struct mlx5_vq_restore_info ri; 124 }; 125 126 static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx) 127 { 128 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) { 129 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 130 return idx < 2; 131 else 132 return idx < 3; 133 } 134 135 return idx <= mvdev->max_idx; 136 } 137 138 static void free_resources(struct mlx5_vdpa_net *ndev); 139 static void init_mvqs(struct mlx5_vdpa_net *ndev); 140 static int setup_driver(struct mlx5_vdpa_dev *mvdev); 141 static void teardown_driver(struct mlx5_vdpa_net *ndev); 142 143 static bool mlx5_vdpa_debug; 144 145 #define MLX5_CVQ_MAX_ENT 16 146 147 #define MLX5_LOG_VIO_FLAG(_feature) \ 148 do { \ 149 if (features & BIT_ULL(_feature)) \ 150 mlx5_vdpa_info(mvdev, "%s\n", #_feature); \ 151 } while (0) 152 153 #define MLX5_LOG_VIO_STAT(_status) \ 154 do { \ 155 if (status & (_status)) \ 156 mlx5_vdpa_info(mvdev, "%s\n", #_status); \ 157 } while (0) 158 159 /* TODO: cross-endian support */ 160 static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev) 161 { 162 return virtio_legacy_is_little_endian() || 163 (mvdev->actual_features & BIT_ULL(VIRTIO_F_VERSION_1)); 164 } 165 166 static u16 mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev *mvdev, __virtio16 val) 167 { 168 return __virtio16_to_cpu(mlx5_vdpa_is_little_endian(mvdev), val); 169 } 170 171 static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val) 172 { 173 return __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), val); 174 } 175 176 static u16 ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev) 177 { 178 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) 179 return 2; 180 181 return mvdev->max_vqs; 182 } 183 184 static bool is_ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev, u16 idx) 185 { 186 return idx == ctrl_vq_idx(mvdev); 187 } 188 189 static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set) 190 { 191 if (status & ~VALID_STATUS_MASK) 192 mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n", 193 status & ~VALID_STATUS_MASK); 194 195 if (!mlx5_vdpa_debug) 196 return; 197 198 mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get"); 199 if (set && !status) { 200 mlx5_vdpa_info(mvdev, "driver resets the device\n"); 201 return; 202 } 203 204 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE); 205 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER); 206 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK); 207 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK); 208 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET); 209 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED); 210 } 211 212 static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set) 213 { 214 if (features & ~VALID_FEATURES_MASK) 215 mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n", 216 features & ~VALID_FEATURES_MASK); 217 218 if (!mlx5_vdpa_debug) 219 return; 220 221 mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads"); 222 if (!features) 223 mlx5_vdpa_info(mvdev, "all feature bits are cleared\n"); 224 225 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM); 226 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM); 227 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 228 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU); 229 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC); 230 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4); 231 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6); 232 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN); 233 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO); 234 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4); 235 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6); 236 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN); 237 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO); 238 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF); 239 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS); 240 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ); 241 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX); 242 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN); 243 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA); 244 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE); 245 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ); 246 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR); 247 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT); 248 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS); 249 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT); 250 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY); 251 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX); 252 MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY); 253 MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT); 254 MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1); 255 MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM); 256 MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED); 257 MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM); 258 MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV); 259 } 260 261 static int create_tis(struct mlx5_vdpa_net *ndev) 262 { 263 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; 264 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 265 void *tisc; 266 int err; 267 268 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 269 MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); 270 err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); 271 if (err) 272 mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err); 273 274 return err; 275 } 276 277 static void destroy_tis(struct mlx5_vdpa_net *ndev) 278 { 279 mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); 280 } 281 282 #define MLX5_VDPA_CQE_SIZE 64 283 #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE) 284 285 static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) 286 { 287 struct mlx5_frag_buf *frag_buf = &buf->frag_buf; 288 u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE; 289 u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE; 290 int err; 291 292 err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, 293 ndev->mvdev.mdev->priv.numa_node); 294 if (err) 295 return err; 296 297 mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc); 298 299 buf->cqe_size = MLX5_VDPA_CQE_SIZE; 300 buf->nent = nent; 301 302 return 0; 303 } 304 305 static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) 306 { 307 struct mlx5_frag_buf *frag_buf = &umem->frag_buf; 308 309 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, 310 ndev->mvdev.mdev->priv.numa_node); 311 } 312 313 static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) 314 { 315 mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); 316 } 317 318 static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n) 319 { 320 return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n); 321 } 322 323 static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf) 324 { 325 struct mlx5_cqe64 *cqe64; 326 void *cqe; 327 int i; 328 329 for (i = 0; i < buf->nent; i++) { 330 cqe = get_cqe(vcq, i); 331 cqe64 = cqe; 332 cqe64->op_own = MLX5_CQE_INVALID << 4; 333 } 334 } 335 336 static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n) 337 { 338 struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1)); 339 340 if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && 341 !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe))) 342 return cqe64; 343 344 return NULL; 345 } 346 347 static void rx_post(struct mlx5_vdpa_qp *vqp, int n) 348 { 349 vqp->head += n; 350 vqp->db.db[0] = cpu_to_be32(vqp->head); 351 } 352 353 static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, 354 struct mlx5_vdpa_virtqueue *mvq, u32 num_ent) 355 { 356 struct mlx5_vdpa_qp *vqp; 357 __be64 *pas; 358 void *qpc; 359 360 vqp = fw ? &mvq->fwqp : &mvq->vqqp; 361 MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); 362 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 363 if (vqp->fw) { 364 /* Firmware QP is allocated by the driver for the firmware's 365 * use so we can skip part of the params as they will be chosen by firmware 366 */ 367 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 368 MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ); 369 MLX5_SET(qpc, qpc, no_sq, 1); 370 return; 371 } 372 373 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 374 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 375 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 376 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 377 MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); 378 MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 379 MLX5_SET(qpc, qpc, no_sq, 1); 380 MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn); 381 MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent)); 382 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 383 pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas); 384 mlx5_fill_page_frag_array(&vqp->frag_buf, pas); 385 } 386 387 static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) 388 { 389 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, 390 num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf, 391 ndev->mvdev.mdev->priv.numa_node); 392 } 393 394 static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 395 { 396 mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); 397 } 398 399 static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 400 struct mlx5_vdpa_qp *vqp) 401 { 402 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 403 int inlen = MLX5_ST_SZ_BYTES(create_qp_in); 404 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 405 void *qpc; 406 void *in; 407 int err; 408 409 if (!vqp->fw) { 410 vqp = &mvq->vqqp; 411 err = rq_buf_alloc(ndev, vqp, mvq->num_ent); 412 if (err) 413 return err; 414 415 err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); 416 if (err) 417 goto err_db; 418 inlen += vqp->frag_buf.npages * sizeof(__be64); 419 } 420 421 in = kzalloc(inlen, GFP_KERNEL); 422 if (!in) { 423 err = -ENOMEM; 424 goto err_kzalloc; 425 } 426 427 qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); 428 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 429 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 430 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 431 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 432 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 433 if (!vqp->fw) 434 MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma); 435 MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 436 err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 437 kfree(in); 438 if (err) 439 goto err_kzalloc; 440 441 vqp->mqp.uid = ndev->mvdev.res.uid; 442 vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn); 443 444 if (!vqp->fw) 445 rx_post(vqp, mvq->num_ent); 446 447 return 0; 448 449 err_kzalloc: 450 if (!vqp->fw) 451 mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 452 err_db: 453 if (!vqp->fw) 454 rq_buf_free(ndev, vqp); 455 456 return err; 457 } 458 459 static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 460 { 461 u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 462 463 MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 464 MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn); 465 MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); 466 if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) 467 mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); 468 if (!vqp->fw) { 469 mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 470 rq_buf_free(ndev, vqp); 471 } 472 } 473 474 static void *next_cqe_sw(struct mlx5_vdpa_cq *cq) 475 { 476 return get_sw_cqe(cq, cq->mcq.cons_index); 477 } 478 479 static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq) 480 { 481 struct mlx5_cqe64 *cqe64; 482 483 cqe64 = next_cqe_sw(vcq); 484 if (!cqe64) 485 return -EAGAIN; 486 487 vcq->mcq.cons_index++; 488 return 0; 489 } 490 491 static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num) 492 { 493 struct mlx5_vdpa_net *ndev = mvq->ndev; 494 struct vdpa_callback *event_cb; 495 496 event_cb = &ndev->event_cbs[mvq->index]; 497 mlx5_cq_set_ci(&mvq->cq.mcq); 498 499 /* make sure CQ cosumer update is visible to the hardware before updating 500 * RX doorbell record. 501 */ 502 dma_wmb(); 503 rx_post(&mvq->vqqp, num); 504 if (event_cb->callback) 505 event_cb->callback(event_cb->private); 506 } 507 508 static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) 509 { 510 struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq); 511 struct mlx5_vdpa_net *ndev = mvq->ndev; 512 void __iomem *uar_page = ndev->mvdev.res.uar->map; 513 int num = 0; 514 515 while (!mlx5_vdpa_poll_one(&mvq->cq)) { 516 num++; 517 if (num > mvq->num_ent / 2) { 518 /* If completions keep coming while we poll, we want to 519 * let the hardware know that we consumed them by 520 * updating the doorbell record. We also let vdpa core 521 * know about this so it passes it on the virtio driver 522 * on the guest. 523 */ 524 mlx5_vdpa_handle_completions(mvq, num); 525 num = 0; 526 } 527 } 528 529 if (num) 530 mlx5_vdpa_handle_completions(mvq, num); 531 532 mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 533 } 534 535 static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) 536 { 537 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 538 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 539 void __iomem *uar_page = ndev->mvdev.res.uar->map; 540 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 541 struct mlx5_vdpa_cq *vcq = &mvq->cq; 542 __be64 *pas; 543 int inlen; 544 void *cqc; 545 void *in; 546 int err; 547 int eqn; 548 549 err = mlx5_db_alloc(mdev, &vcq->db); 550 if (err) 551 return err; 552 553 vcq->mcq.set_ci_db = vcq->db.db; 554 vcq->mcq.arm_db = vcq->db.db + 1; 555 vcq->mcq.cqe_sz = 64; 556 557 err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); 558 if (err) 559 goto err_db; 560 561 cq_frag_buf_init(vcq, &vcq->buf); 562 563 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 564 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages; 565 in = kzalloc(inlen, GFP_KERNEL); 566 if (!in) { 567 err = -ENOMEM; 568 goto err_vzalloc; 569 } 570 571 MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); 572 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 573 mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas); 574 575 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 576 MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 577 578 /* Use vector 0 by default. Consider adding code to choose least used 579 * vector. 580 */ 581 err = mlx5_vector2eqn(mdev, 0, &eqn); 582 if (err) 583 goto err_vec; 584 585 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 586 MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent)); 587 MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); 588 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 589 MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma); 590 591 err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out)); 592 if (err) 593 goto err_vec; 594 595 vcq->mcq.comp = mlx5_vdpa_cq_comp; 596 vcq->cqe = num_ent; 597 vcq->mcq.set_ci_db = vcq->db.db; 598 vcq->mcq.arm_db = vcq->db.db + 1; 599 mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 600 kfree(in); 601 return 0; 602 603 err_vec: 604 kfree(in); 605 err_vzalloc: 606 cq_frag_buf_free(ndev, &vcq->buf); 607 err_db: 608 mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 609 return err; 610 } 611 612 static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) 613 { 614 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 615 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 616 struct mlx5_vdpa_cq *vcq = &mvq->cq; 617 618 if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) { 619 mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); 620 return; 621 } 622 cq_frag_buf_free(ndev, &vcq->buf); 623 mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 624 } 625 626 static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, 627 struct mlx5_vdpa_umem **umemp) 628 { 629 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 630 int p_a; 631 int p_b; 632 633 switch (num) { 634 case 1: 635 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a); 636 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b); 637 *umemp = &mvq->umem1; 638 break; 639 case 2: 640 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a); 641 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b); 642 *umemp = &mvq->umem2; 643 break; 644 case 3: 645 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a); 646 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b); 647 *umemp = &mvq->umem3; 648 break; 649 } 650 (*umemp)->size = p_a * mvq->num_ent + p_b; 651 } 652 653 static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) 654 { 655 mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); 656 } 657 658 static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 659 { 660 int inlen; 661 u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {}; 662 void *um; 663 void *in; 664 int err; 665 __be64 *pas; 666 struct mlx5_vdpa_umem *umem; 667 668 set_umem_size(ndev, mvq, num, &umem); 669 err = umem_frag_buf_alloc(ndev, umem, umem->size); 670 if (err) 671 return err; 672 673 inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages; 674 675 in = kzalloc(inlen, GFP_KERNEL); 676 if (!in) { 677 err = -ENOMEM; 678 goto err_in; 679 } 680 681 MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM); 682 MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); 683 um = MLX5_ADDR_OF(create_umem_in, in, umem); 684 MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 685 MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages); 686 687 pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]); 688 mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW); 689 690 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 691 if (err) { 692 mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); 693 goto err_cmd; 694 } 695 696 kfree(in); 697 umem->id = MLX5_GET(create_umem_out, out, umem_id); 698 699 return 0; 700 701 err_cmd: 702 kfree(in); 703 err_in: 704 umem_frag_buf_free(ndev, umem); 705 return err; 706 } 707 708 static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 709 { 710 u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {}; 711 u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {}; 712 struct mlx5_vdpa_umem *umem; 713 714 switch (num) { 715 case 1: 716 umem = &mvq->umem1; 717 break; 718 case 2: 719 umem = &mvq->umem2; 720 break; 721 case 3: 722 umem = &mvq->umem3; 723 break; 724 } 725 726 MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM); 727 MLX5_SET(destroy_umem_in, in, umem_id, umem->id); 728 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 729 return; 730 731 umem_frag_buf_free(ndev, umem); 732 } 733 734 static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 735 { 736 int num; 737 int err; 738 739 for (num = 1; num <= 3; num++) { 740 err = create_umem(ndev, mvq, num); 741 if (err) 742 goto err_umem; 743 } 744 return 0; 745 746 err_umem: 747 for (num--; num > 0; num--) 748 umem_destroy(ndev, mvq, num); 749 750 return err; 751 } 752 753 static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 754 { 755 int num; 756 757 for (num = 3; num > 0; num--) 758 umem_destroy(ndev, mvq, num); 759 } 760 761 static int get_queue_type(struct mlx5_vdpa_net *ndev) 762 { 763 u32 type_mask; 764 765 type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); 766 767 /* prefer split queue */ 768 if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT) 769 return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT; 770 771 WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED)); 772 773 return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED; 774 } 775 776 static bool vq_is_tx(u16 idx) 777 { 778 return idx % 2; 779 } 780 781 static u16 get_features_12_3(u64 features) 782 { 783 return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) | 784 (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) | 785 (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) | 786 (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6); 787 } 788 789 static bool counters_supported(const struct mlx5_vdpa_dev *mvdev) 790 { 791 return MLX5_CAP_GEN_64(mvdev->mdev, general_obj_types) & 792 BIT_ULL(MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 793 } 794 795 static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 796 { 797 int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); 798 u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {}; 799 void *obj_context; 800 void *cmd_hdr; 801 void *vq_ctx; 802 void *in; 803 int err; 804 805 err = umems_create(ndev, mvq); 806 if (err) 807 return err; 808 809 in = kzalloc(inlen, GFP_KERNEL); 810 if (!in) { 811 err = -ENOMEM; 812 goto err_alloc; 813 } 814 815 cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr); 816 817 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 818 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 819 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 820 821 obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context); 822 MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); 823 MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); 824 MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, 825 get_features_12_3(ndev->mvdev.actual_features)); 826 vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context); 827 MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); 828 829 if (vq_is_tx(mvq->index)) 830 MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); 831 832 MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 833 MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); 834 MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 835 MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); 836 MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 837 !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); 838 MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); 839 MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); 840 MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); 841 MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey); 842 MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); 843 MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); 844 MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id); 845 MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem2.size); 846 MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id); 847 MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem3.size); 848 MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); 849 if (counters_supported(&ndev->mvdev)) 850 MLX5_SET(virtio_q, vq_ctx, counter_set_id, mvq->counter_set_id); 851 852 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 853 if (err) 854 goto err_cmd; 855 856 mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT; 857 kfree(in); 858 mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 859 860 return 0; 861 862 err_cmd: 863 kfree(in); 864 err_alloc: 865 umems_destroy(ndev, mvq); 866 return err; 867 } 868 869 static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 870 { 871 u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {}; 872 u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {}; 873 874 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode, 875 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 876 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id); 877 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); 878 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type, 879 MLX5_OBJ_TYPE_VIRTIO_NET_Q); 880 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { 881 mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); 882 return; 883 } 884 mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; 885 umems_destroy(ndev, mvq); 886 } 887 888 static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 889 { 890 return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn; 891 } 892 893 static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 894 { 895 return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn; 896 } 897 898 static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, 899 int *outlen, u32 qpn, u32 rqpn) 900 { 901 void *qpc; 902 void *pp; 903 904 switch (cmd) { 905 case MLX5_CMD_OP_2RST_QP: 906 *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in); 907 *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out); 908 *in = kzalloc(*inlen, GFP_KERNEL); 909 *out = kzalloc(*outlen, GFP_KERNEL); 910 if (!*in || !*out) 911 goto outerr; 912 913 MLX5_SET(qp_2rst_in, *in, opcode, cmd); 914 MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); 915 MLX5_SET(qp_2rst_in, *in, qpn, qpn); 916 break; 917 case MLX5_CMD_OP_RST2INIT_QP: 918 *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in); 919 *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out); 920 *in = kzalloc(*inlen, GFP_KERNEL); 921 *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL); 922 if (!*in || !*out) 923 goto outerr; 924 925 MLX5_SET(rst2init_qp_in, *in, opcode, cmd); 926 MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); 927 MLX5_SET(rst2init_qp_in, *in, qpn, qpn); 928 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 929 MLX5_SET(qpc, qpc, remote_qpn, rqpn); 930 MLX5_SET(qpc, qpc, rwe, 1); 931 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 932 MLX5_SET(ads, pp, vhca_port_num, 1); 933 break; 934 case MLX5_CMD_OP_INIT2RTR_QP: 935 *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in); 936 *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out); 937 *in = kzalloc(*inlen, GFP_KERNEL); 938 *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL); 939 if (!*in || !*out) 940 goto outerr; 941 942 MLX5_SET(init2rtr_qp_in, *in, opcode, cmd); 943 MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); 944 MLX5_SET(init2rtr_qp_in, *in, qpn, qpn); 945 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 946 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 947 MLX5_SET(qpc, qpc, log_msg_max, 30); 948 MLX5_SET(qpc, qpc, remote_qpn, rqpn); 949 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 950 MLX5_SET(ads, pp, fl, 1); 951 break; 952 case MLX5_CMD_OP_RTR2RTS_QP: 953 *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in); 954 *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out); 955 *in = kzalloc(*inlen, GFP_KERNEL); 956 *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL); 957 if (!*in || !*out) 958 goto outerr; 959 960 MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd); 961 MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); 962 MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn); 963 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 964 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 965 MLX5_SET(ads, pp, ack_timeout, 14); 966 MLX5_SET(qpc, qpc, retry_count, 7); 967 MLX5_SET(qpc, qpc, rnr_retry, 7); 968 break; 969 default: 970 goto outerr_nullify; 971 } 972 973 return; 974 975 outerr: 976 kfree(*in); 977 kfree(*out); 978 outerr_nullify: 979 *in = NULL; 980 *out = NULL; 981 } 982 983 static void free_inout(void *in, void *out) 984 { 985 kfree(in); 986 kfree(out); 987 } 988 989 /* Two QPs are used by each virtqueue. One is used by the driver and one by 990 * firmware. The fw argument indicates whether the subjected QP is the one used 991 * by firmware. 992 */ 993 static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) 994 { 995 int outlen; 996 int inlen; 997 void *out; 998 void *in; 999 int err; 1000 1001 alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); 1002 if (!in || !out) 1003 return -ENOMEM; 1004 1005 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); 1006 free_inout(in, out); 1007 return err; 1008 } 1009 1010 static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1011 { 1012 int err; 1013 1014 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); 1015 if (err) 1016 return err; 1017 1018 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); 1019 if (err) 1020 return err; 1021 1022 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); 1023 if (err) 1024 return err; 1025 1026 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); 1027 if (err) 1028 return err; 1029 1030 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); 1031 if (err) 1032 return err; 1033 1034 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); 1035 if (err) 1036 return err; 1037 1038 return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); 1039 } 1040 1041 struct mlx5_virtq_attr { 1042 u8 state; 1043 u16 available_index; 1044 u16 used_index; 1045 }; 1046 1047 static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 1048 struct mlx5_virtq_attr *attr) 1049 { 1050 int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out); 1051 u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {}; 1052 void *out; 1053 void *obj_context; 1054 void *cmd_hdr; 1055 int err; 1056 1057 out = kzalloc(outlen, GFP_KERNEL); 1058 if (!out) 1059 return -ENOMEM; 1060 1061 cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr); 1062 1063 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 1064 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 1065 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 1066 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 1067 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen); 1068 if (err) 1069 goto err_cmd; 1070 1071 obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context); 1072 memset(attr, 0, sizeof(*attr)); 1073 attr->state = MLX5_GET(virtio_net_q_object, obj_context, state); 1074 attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index); 1075 attr->used_index = MLX5_GET(virtio_net_q_object, obj_context, hw_used_index); 1076 kfree(out); 1077 return 0; 1078 1079 err_cmd: 1080 kfree(out); 1081 return err; 1082 } 1083 1084 static bool is_valid_state_change(int oldstate, int newstate) 1085 { 1086 switch (oldstate) { 1087 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT: 1088 return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY; 1089 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY: 1090 return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND; 1091 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND: 1092 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR: 1093 default: 1094 return false; 1095 } 1096 } 1097 1098 static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) 1099 { 1100 int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); 1101 u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {}; 1102 void *obj_context; 1103 void *cmd_hdr; 1104 void *in; 1105 int err; 1106 1107 if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_NONE) 1108 return 0; 1109 1110 if (!is_valid_state_change(mvq->fw_state, state)) 1111 return -EINVAL; 1112 1113 in = kzalloc(inlen, GFP_KERNEL); 1114 if (!in) 1115 return -ENOMEM; 1116 1117 cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr); 1118 1119 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); 1120 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 1121 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 1122 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 1123 1124 obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context); 1125 MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select, 1126 MLX5_VIRTQ_MODIFY_MASK_STATE); 1127 MLX5_SET(virtio_net_q_object, obj_context, state, state); 1128 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 1129 kfree(in); 1130 if (!err) 1131 mvq->fw_state = state; 1132 1133 return err; 1134 } 1135 1136 static int counter_set_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1137 { 1138 u32 in[MLX5_ST_SZ_DW(create_virtio_q_counters_in)] = {}; 1139 u32 out[MLX5_ST_SZ_DW(create_virtio_q_counters_out)] = {}; 1140 void *cmd_hdr; 1141 int err; 1142 1143 if (!counters_supported(&ndev->mvdev)) 1144 return 0; 1145 1146 cmd_hdr = MLX5_ADDR_OF(create_virtio_q_counters_in, in, hdr); 1147 1148 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 1149 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 1150 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 1151 1152 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 1153 if (err) 1154 return err; 1155 1156 mvq->counter_set_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 1157 1158 return 0; 1159 } 1160 1161 static void counter_set_dealloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1162 { 1163 u32 in[MLX5_ST_SZ_DW(destroy_virtio_q_counters_in)] = {}; 1164 u32 out[MLX5_ST_SZ_DW(destroy_virtio_q_counters_out)] = {}; 1165 1166 if (!counters_supported(&ndev->mvdev)) 1167 return; 1168 1169 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 1170 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_id, mvq->counter_set_id); 1171 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.uid, ndev->mvdev.res.uid); 1172 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 1173 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 1174 mlx5_vdpa_warn(&ndev->mvdev, "dealloc counter set 0x%x\n", mvq->counter_set_id); 1175 } 1176 1177 static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1178 { 1179 u16 idx = mvq->index; 1180 int err; 1181 1182 if (!mvq->num_ent) 1183 return 0; 1184 1185 if (mvq->initialized) 1186 return 0; 1187 1188 err = cq_create(ndev, idx, mvq->num_ent); 1189 if (err) 1190 return err; 1191 1192 err = qp_create(ndev, mvq, &mvq->fwqp); 1193 if (err) 1194 goto err_fwqp; 1195 1196 err = qp_create(ndev, mvq, &mvq->vqqp); 1197 if (err) 1198 goto err_vqqp; 1199 1200 err = connect_qps(ndev, mvq); 1201 if (err) 1202 goto err_connect; 1203 1204 err = counter_set_alloc(ndev, mvq); 1205 if (err) 1206 goto err_counter; 1207 1208 err = create_virtqueue(ndev, mvq); 1209 if (err) 1210 goto err_connect; 1211 1212 if (mvq->ready) { 1213 err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 1214 if (err) { 1215 mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", 1216 idx, err); 1217 goto err_connect; 1218 } 1219 } 1220 1221 mvq->initialized = true; 1222 return 0; 1223 1224 err_connect: 1225 counter_set_dealloc(ndev, mvq); 1226 err_counter: 1227 qp_destroy(ndev, &mvq->vqqp); 1228 err_vqqp: 1229 qp_destroy(ndev, &mvq->fwqp); 1230 err_fwqp: 1231 cq_destroy(ndev, idx); 1232 return err; 1233 } 1234 1235 static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1236 { 1237 struct mlx5_virtq_attr attr; 1238 1239 if (!mvq->initialized) 1240 return; 1241 1242 if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 1243 return; 1244 1245 if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) 1246 mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); 1247 1248 if (query_virtqueue(ndev, mvq, &attr)) { 1249 mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); 1250 return; 1251 } 1252 mvq->avail_idx = attr.available_index; 1253 mvq->used_idx = attr.used_index; 1254 } 1255 1256 static void suspend_vqs(struct mlx5_vdpa_net *ndev) 1257 { 1258 int i; 1259 1260 for (i = 0; i < ndev->mvdev.max_vqs; i++) 1261 suspend_vq(ndev, &ndev->vqs[i]); 1262 } 1263 1264 static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1265 { 1266 if (!mvq->initialized) 1267 return; 1268 1269 suspend_vq(ndev, mvq); 1270 destroy_virtqueue(ndev, mvq); 1271 counter_set_dealloc(ndev, mvq); 1272 qp_destroy(ndev, &mvq->vqqp); 1273 qp_destroy(ndev, &mvq->fwqp); 1274 cq_destroy(ndev, mvq->index); 1275 mvq->initialized = false; 1276 } 1277 1278 static int create_rqt(struct mlx5_vdpa_net *ndev) 1279 { 1280 int rqt_table_size = roundup_pow_of_two(ndev->rqt_size); 1281 int act_sz = roundup_pow_of_two(ndev->cur_num_vqs / 2); 1282 __be32 *list; 1283 void *rqtc; 1284 int inlen; 1285 void *in; 1286 int i, j; 1287 int err; 1288 1289 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + rqt_table_size * MLX5_ST_SZ_BYTES(rq_num); 1290 in = kzalloc(inlen, GFP_KERNEL); 1291 if (!in) 1292 return -ENOMEM; 1293 1294 MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); 1295 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 1296 1297 MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 1298 MLX5_SET(rqtc, rqtc, rqt_max_size, rqt_table_size); 1299 list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1300 for (i = 0, j = 0; i < act_sz; i++, j += 2) 1301 list[i] = cpu_to_be32(ndev->vqs[j % ndev->cur_num_vqs].virtq_id); 1302 1303 MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz); 1304 err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); 1305 kfree(in); 1306 if (err) 1307 return err; 1308 1309 return 0; 1310 } 1311 1312 #define MLX5_MODIFY_RQT_NUM_RQS ((u64)1) 1313 1314 static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) 1315 { 1316 int act_sz = roundup_pow_of_two(num / 2); 1317 __be32 *list; 1318 void *rqtc; 1319 int inlen; 1320 void *in; 1321 int i, j; 1322 int err; 1323 1324 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + act_sz * MLX5_ST_SZ_BYTES(rq_num); 1325 in = kzalloc(inlen, GFP_KERNEL); 1326 if (!in) 1327 return -ENOMEM; 1328 1329 MLX5_SET(modify_rqt_in, in, uid, ndev->mvdev.res.uid); 1330 MLX5_SET64(modify_rqt_in, in, bitmask, MLX5_MODIFY_RQT_NUM_RQS); 1331 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 1332 MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 1333 1334 list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1335 for (i = 0, j = 0; i < act_sz; i++, j = j + 2) 1336 list[i] = cpu_to_be32(ndev->vqs[j % num].virtq_id); 1337 1338 MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz); 1339 err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn); 1340 kfree(in); 1341 if (err) 1342 return err; 1343 1344 return 0; 1345 } 1346 1347 static void destroy_rqt(struct mlx5_vdpa_net *ndev) 1348 { 1349 mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); 1350 } 1351 1352 static int create_tir(struct mlx5_vdpa_net *ndev) 1353 { 1354 #define HASH_IP_L4PORTS \ 1355 (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \ 1356 MLX5_HASH_FIELD_SEL_L4_DPORT) 1357 static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 1358 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 1359 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 1360 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 1361 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a }; 1362 void *rss_key; 1363 void *outer; 1364 void *tirc; 1365 void *in; 1366 int err; 1367 1368 in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL); 1369 if (!in) 1370 return -ENOMEM; 1371 1372 MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); 1373 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 1374 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 1375 1376 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 1377 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ); 1378 rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 1379 memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key)); 1380 1381 outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 1382 MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); 1383 MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); 1384 MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS); 1385 1386 MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); 1387 MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); 1388 1389 err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); 1390 kfree(in); 1391 if (err) 1392 return err; 1393 1394 mlx5_vdpa_add_tirn(ndev); 1395 return err; 1396 } 1397 1398 static void destroy_tir(struct mlx5_vdpa_net *ndev) 1399 { 1400 mlx5_vdpa_remove_tirn(ndev); 1401 mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); 1402 } 1403 1404 #define MAX_STEERING_ENT 0x8000 1405 #define MAX_STEERING_GROUPS 2 1406 1407 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1408 #define NUM_DESTS 2 1409 #else 1410 #define NUM_DESTS 1 1411 #endif 1412 1413 static int add_steering_counters(struct mlx5_vdpa_net *ndev, 1414 struct macvlan_node *node, 1415 struct mlx5_flow_act *flow_act, 1416 struct mlx5_flow_destination *dests) 1417 { 1418 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1419 int err; 1420 1421 node->ucast_counter.counter = mlx5_fc_create(ndev->mvdev.mdev, false); 1422 if (IS_ERR(node->ucast_counter.counter)) 1423 return PTR_ERR(node->ucast_counter.counter); 1424 1425 node->mcast_counter.counter = mlx5_fc_create(ndev->mvdev.mdev, false); 1426 if (IS_ERR(node->mcast_counter.counter)) { 1427 err = PTR_ERR(node->mcast_counter.counter); 1428 goto err_mcast_counter; 1429 } 1430 1431 dests[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1432 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 1433 return 0; 1434 1435 err_mcast_counter: 1436 mlx5_fc_destroy(ndev->mvdev.mdev, node->ucast_counter.counter); 1437 return err; 1438 #else 1439 return 0; 1440 #endif 1441 } 1442 1443 static void remove_steering_counters(struct mlx5_vdpa_net *ndev, 1444 struct macvlan_node *node) 1445 { 1446 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1447 mlx5_fc_destroy(ndev->mvdev.mdev, node->mcast_counter.counter); 1448 mlx5_fc_destroy(ndev->mvdev.mdev, node->ucast_counter.counter); 1449 #endif 1450 } 1451 1452 static int mlx5_vdpa_add_mac_vlan_rules(struct mlx5_vdpa_net *ndev, u8 *mac, 1453 struct macvlan_node *node) 1454 { 1455 struct mlx5_flow_destination dests[NUM_DESTS] = {}; 1456 struct mlx5_flow_act flow_act = {}; 1457 struct mlx5_flow_spec *spec; 1458 void *headers_c; 1459 void *headers_v; 1460 u8 *dmac_c; 1461 u8 *dmac_v; 1462 int err; 1463 u16 vid; 1464 1465 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1466 if (!spec) 1467 return -ENOMEM; 1468 1469 vid = key2vid(node->macvlan); 1470 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1471 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers); 1472 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); 1473 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, outer_headers.dmac_47_16); 1474 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16); 1475 eth_broadcast_addr(dmac_c); 1476 ether_addr_copy(dmac_v, mac); 1477 if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VLAN)) { 1478 MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1); 1479 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, first_vid); 1480 } 1481 if (node->tagged) { 1482 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); 1483 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, vid); 1484 } 1485 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1486 dests[0].type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1487 dests[0].tir_num = ndev->res.tirn; 1488 err = add_steering_counters(ndev, node, &flow_act, dests); 1489 if (err) 1490 goto out_free; 1491 1492 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1493 dests[1].counter_id = mlx5_fc_id(node->ucast_counter.counter); 1494 #endif 1495 node->ucast_rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dests, NUM_DESTS); 1496 if (IS_ERR(node->ucast_rule)) { 1497 err = PTR_ERR(node->ucast_rule); 1498 goto err_ucast; 1499 } 1500 1501 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1502 dests[1].counter_id = mlx5_fc_id(node->mcast_counter.counter); 1503 #endif 1504 1505 memset(dmac_c, 0, ETH_ALEN); 1506 memset(dmac_v, 0, ETH_ALEN); 1507 dmac_c[0] = 1; 1508 dmac_v[0] = 1; 1509 node->mcast_rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dests, NUM_DESTS); 1510 if (IS_ERR(node->mcast_rule)) { 1511 err = PTR_ERR(node->mcast_rule); 1512 goto err_mcast; 1513 } 1514 kvfree(spec); 1515 mlx5_vdpa_add_rx_counters(ndev, node); 1516 return 0; 1517 1518 err_mcast: 1519 mlx5_del_flow_rules(node->ucast_rule); 1520 err_ucast: 1521 remove_steering_counters(ndev, node); 1522 out_free: 1523 kvfree(spec); 1524 return err; 1525 } 1526 1527 static void mlx5_vdpa_del_mac_vlan_rules(struct mlx5_vdpa_net *ndev, 1528 struct macvlan_node *node) 1529 { 1530 mlx5_vdpa_remove_rx_counters(ndev, node); 1531 mlx5_del_flow_rules(node->ucast_rule); 1532 mlx5_del_flow_rules(node->mcast_rule); 1533 } 1534 1535 static u64 search_val(u8 *mac, u16 vlan, bool tagged) 1536 { 1537 u64 val; 1538 1539 if (!tagged) 1540 vlan = MLX5V_UNTAGGED; 1541 1542 val = (u64)vlan << 48 | 1543 (u64)mac[0] << 40 | 1544 (u64)mac[1] << 32 | 1545 (u64)mac[2] << 24 | 1546 (u64)mac[3] << 16 | 1547 (u64)mac[4] << 8 | 1548 (u64)mac[5]; 1549 1550 return val; 1551 } 1552 1553 static struct macvlan_node *mac_vlan_lookup(struct mlx5_vdpa_net *ndev, u64 value) 1554 { 1555 struct macvlan_node *pos; 1556 u32 idx; 1557 1558 idx = hash_64(value, 8); // tbd 8 1559 hlist_for_each_entry(pos, &ndev->macvlan_hash[idx], hlist) { 1560 if (pos->macvlan == value) 1561 return pos; 1562 } 1563 return NULL; 1564 } 1565 1566 static int mac_vlan_add(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vid, bool tagged) 1567 { 1568 struct macvlan_node *ptr; 1569 u64 val; 1570 u32 idx; 1571 int err; 1572 1573 val = search_val(mac, vid, tagged); 1574 if (mac_vlan_lookup(ndev, val)) 1575 return -EEXIST; 1576 1577 ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); 1578 if (!ptr) 1579 return -ENOMEM; 1580 1581 ptr->tagged = tagged; 1582 ptr->macvlan = val; 1583 ptr->ndev = ndev; 1584 err = mlx5_vdpa_add_mac_vlan_rules(ndev, ndev->config.mac, ptr); 1585 if (err) 1586 goto err_add; 1587 1588 idx = hash_64(val, 8); 1589 hlist_add_head(&ptr->hlist, &ndev->macvlan_hash[idx]); 1590 return 0; 1591 1592 err_add: 1593 kfree(ptr); 1594 return err; 1595 } 1596 1597 static void mac_vlan_del(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vlan, bool tagged) 1598 { 1599 struct macvlan_node *ptr; 1600 1601 ptr = mac_vlan_lookup(ndev, search_val(mac, vlan, tagged)); 1602 if (!ptr) 1603 return; 1604 1605 hlist_del(&ptr->hlist); 1606 mlx5_vdpa_del_mac_vlan_rules(ndev, ptr); 1607 remove_steering_counters(ndev, ptr); 1608 kfree(ptr); 1609 } 1610 1611 static void clear_mac_vlan_table(struct mlx5_vdpa_net *ndev) 1612 { 1613 struct macvlan_node *pos; 1614 struct hlist_node *n; 1615 int i; 1616 1617 for (i = 0; i < MLX5V_MACVLAN_SIZE; i++) { 1618 hlist_for_each_entry_safe(pos, n, &ndev->macvlan_hash[i], hlist) { 1619 hlist_del(&pos->hlist); 1620 mlx5_vdpa_del_mac_vlan_rules(ndev, pos); 1621 remove_steering_counters(ndev, pos); 1622 kfree(pos); 1623 } 1624 } 1625 } 1626 1627 static int setup_steering(struct mlx5_vdpa_net *ndev) 1628 { 1629 struct mlx5_flow_table_attr ft_attr = {}; 1630 struct mlx5_flow_namespace *ns; 1631 int err; 1632 1633 ft_attr.max_fte = MAX_STEERING_ENT; 1634 ft_attr.autogroup.max_num_groups = MAX_STEERING_GROUPS; 1635 1636 ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); 1637 if (!ns) { 1638 mlx5_vdpa_warn(&ndev->mvdev, "failed to get flow namespace\n"); 1639 return -EOPNOTSUPP; 1640 } 1641 1642 ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 1643 if (IS_ERR(ndev->rxft)) { 1644 mlx5_vdpa_warn(&ndev->mvdev, "failed to create flow table\n"); 1645 return PTR_ERR(ndev->rxft); 1646 } 1647 mlx5_vdpa_add_rx_flow_table(ndev); 1648 1649 err = mac_vlan_add(ndev, ndev->config.mac, 0, false); 1650 if (err) 1651 goto err_add; 1652 1653 return 0; 1654 1655 err_add: 1656 mlx5_vdpa_remove_rx_flow_table(ndev); 1657 mlx5_destroy_flow_table(ndev->rxft); 1658 return err; 1659 } 1660 1661 static void teardown_steering(struct mlx5_vdpa_net *ndev) 1662 { 1663 clear_mac_vlan_table(ndev); 1664 mlx5_vdpa_remove_rx_flow_table(ndev); 1665 mlx5_destroy_flow_table(ndev->rxft); 1666 } 1667 1668 static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1669 { 1670 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1671 struct mlx5_control_vq *cvq = &mvdev->cvq; 1672 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1673 struct mlx5_core_dev *pfmdev; 1674 size_t read; 1675 u8 mac[ETH_ALEN], mac_back[ETH_ALEN]; 1676 1677 pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 1678 switch (cmd) { 1679 case VIRTIO_NET_CTRL_MAC_ADDR_SET: 1680 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)mac, ETH_ALEN); 1681 if (read != ETH_ALEN) 1682 break; 1683 1684 if (!memcmp(ndev->config.mac, mac, 6)) { 1685 status = VIRTIO_NET_OK; 1686 break; 1687 } 1688 1689 if (is_zero_ether_addr(mac)) 1690 break; 1691 1692 if (!is_zero_ether_addr(ndev->config.mac)) { 1693 if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 1694 mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n", 1695 ndev->config.mac); 1696 break; 1697 } 1698 } 1699 1700 if (mlx5_mpfs_add_mac(pfmdev, mac)) { 1701 mlx5_vdpa_warn(mvdev, "failed to insert new MAC %pM into MPFS table\n", 1702 mac); 1703 break; 1704 } 1705 1706 /* backup the original mac address so that if failed to add the forward rules 1707 * we could restore it 1708 */ 1709 memcpy(mac_back, ndev->config.mac, ETH_ALEN); 1710 1711 memcpy(ndev->config.mac, mac, ETH_ALEN); 1712 1713 /* Need recreate the flow table entry, so that the packet could forward back 1714 */ 1715 mac_vlan_del(ndev, mac_back, 0, false); 1716 1717 if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) { 1718 mlx5_vdpa_warn(mvdev, "failed to insert forward rules, try to restore\n"); 1719 1720 /* Although it hardly run here, we still need double check */ 1721 if (is_zero_ether_addr(mac_back)) { 1722 mlx5_vdpa_warn(mvdev, "restore mac failed: Original MAC is zero\n"); 1723 break; 1724 } 1725 1726 /* Try to restore original mac address to MFPS table, and try to restore 1727 * the forward rule entry. 1728 */ 1729 if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 1730 mlx5_vdpa_warn(mvdev, "restore mac failed: delete MAC %pM from MPFS table failed\n", 1731 ndev->config.mac); 1732 } 1733 1734 if (mlx5_mpfs_add_mac(pfmdev, mac_back)) { 1735 mlx5_vdpa_warn(mvdev, "restore mac failed: insert old MAC %pM into MPFS table failed\n", 1736 mac_back); 1737 } 1738 1739 memcpy(ndev->config.mac, mac_back, ETH_ALEN); 1740 1741 if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) 1742 mlx5_vdpa_warn(mvdev, "restore forward rules failed: insert forward rules failed\n"); 1743 1744 break; 1745 } 1746 1747 status = VIRTIO_NET_OK; 1748 break; 1749 1750 default: 1751 break; 1752 } 1753 1754 return status; 1755 } 1756 1757 static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps) 1758 { 1759 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1760 int cur_qps = ndev->cur_num_vqs / 2; 1761 int err; 1762 int i; 1763 1764 if (cur_qps > newqps) { 1765 err = modify_rqt(ndev, 2 * newqps); 1766 if (err) 1767 return err; 1768 1769 for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) 1770 teardown_vq(ndev, &ndev->vqs[i]); 1771 1772 ndev->cur_num_vqs = 2 * newqps; 1773 } else { 1774 ndev->cur_num_vqs = 2 * newqps; 1775 for (i = cur_qps * 2; i < 2 * newqps; i++) { 1776 err = setup_vq(ndev, &ndev->vqs[i]); 1777 if (err) 1778 goto clean_added; 1779 } 1780 err = modify_rqt(ndev, 2 * newqps); 1781 if (err) 1782 goto clean_added; 1783 } 1784 return 0; 1785 1786 clean_added: 1787 for (--i; i >= 2 * cur_qps; --i) 1788 teardown_vq(ndev, &ndev->vqs[i]); 1789 1790 ndev->cur_num_vqs = 2 * cur_qps; 1791 1792 return err; 1793 } 1794 1795 static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1796 { 1797 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1798 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1799 struct mlx5_control_vq *cvq = &mvdev->cvq; 1800 struct virtio_net_ctrl_mq mq; 1801 size_t read; 1802 u16 newqps; 1803 1804 switch (cmd) { 1805 case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: 1806 /* This mq feature check aligns with pre-existing userspace 1807 * implementation. 1808 * 1809 * Without it, an untrusted driver could fake a multiqueue config 1810 * request down to a non-mq device that may cause kernel to 1811 * panic due to uninitialized resources for extra vqs. Even with 1812 * a well behaving guest driver, it is not expected to allow 1813 * changing the number of vqs on a non-mq device. 1814 */ 1815 if (!MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) 1816 break; 1817 1818 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)&mq, sizeof(mq)); 1819 if (read != sizeof(mq)) 1820 break; 1821 1822 newqps = mlx5vdpa16_to_cpu(mvdev, mq.virtqueue_pairs); 1823 if (newqps < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 1824 newqps > ndev->rqt_size) 1825 break; 1826 1827 if (ndev->cur_num_vqs == 2 * newqps) { 1828 status = VIRTIO_NET_OK; 1829 break; 1830 } 1831 1832 if (!change_num_qps(mvdev, newqps)) 1833 status = VIRTIO_NET_OK; 1834 1835 break; 1836 default: 1837 break; 1838 } 1839 1840 return status; 1841 } 1842 1843 static virtio_net_ctrl_ack handle_ctrl_vlan(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1844 { 1845 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1846 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1847 struct mlx5_control_vq *cvq = &mvdev->cvq; 1848 __virtio16 vlan; 1849 size_t read; 1850 u16 id; 1851 1852 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VLAN))) 1853 return status; 1854 1855 switch (cmd) { 1856 case VIRTIO_NET_CTRL_VLAN_ADD: 1857 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &vlan, sizeof(vlan)); 1858 if (read != sizeof(vlan)) 1859 break; 1860 1861 id = mlx5vdpa16_to_cpu(mvdev, vlan); 1862 if (mac_vlan_add(ndev, ndev->config.mac, id, true)) 1863 break; 1864 1865 status = VIRTIO_NET_OK; 1866 break; 1867 case VIRTIO_NET_CTRL_VLAN_DEL: 1868 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &vlan, sizeof(vlan)); 1869 if (read != sizeof(vlan)) 1870 break; 1871 1872 id = mlx5vdpa16_to_cpu(mvdev, vlan); 1873 mac_vlan_del(ndev, ndev->config.mac, id, true); 1874 status = VIRTIO_NET_OK; 1875 break; 1876 default: 1877 break; 1878 } 1879 1880 return status; 1881 } 1882 1883 static void mlx5_cvq_kick_handler(struct work_struct *work) 1884 { 1885 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1886 struct virtio_net_ctrl_hdr ctrl; 1887 struct mlx5_vdpa_wq_ent *wqent; 1888 struct mlx5_vdpa_dev *mvdev; 1889 struct mlx5_control_vq *cvq; 1890 struct mlx5_vdpa_net *ndev; 1891 size_t read, write; 1892 int err; 1893 1894 wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 1895 mvdev = wqent->mvdev; 1896 ndev = to_mlx5_vdpa_ndev(mvdev); 1897 cvq = &mvdev->cvq; 1898 1899 down_write(&ndev->reslock); 1900 1901 if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 1902 goto out; 1903 1904 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 1905 goto out; 1906 1907 if (!cvq->ready) 1908 goto out; 1909 1910 while (true) { 1911 err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head, 1912 GFP_ATOMIC); 1913 if (err <= 0) 1914 break; 1915 1916 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &ctrl, sizeof(ctrl)); 1917 if (read != sizeof(ctrl)) 1918 break; 1919 1920 cvq->received_desc++; 1921 switch (ctrl.class) { 1922 case VIRTIO_NET_CTRL_MAC: 1923 status = handle_ctrl_mac(mvdev, ctrl.cmd); 1924 break; 1925 case VIRTIO_NET_CTRL_MQ: 1926 status = handle_ctrl_mq(mvdev, ctrl.cmd); 1927 break; 1928 case VIRTIO_NET_CTRL_VLAN: 1929 status = handle_ctrl_vlan(mvdev, ctrl.cmd); 1930 break; 1931 default: 1932 break; 1933 } 1934 1935 /* Make sure data is written before advancing index */ 1936 smp_wmb(); 1937 1938 write = vringh_iov_push_iotlb(&cvq->vring, &cvq->wiov, &status, sizeof(status)); 1939 vringh_complete_iotlb(&cvq->vring, cvq->head, write); 1940 vringh_kiov_cleanup(&cvq->riov); 1941 vringh_kiov_cleanup(&cvq->wiov); 1942 1943 if (vringh_need_notify_iotlb(&cvq->vring)) 1944 vringh_notify(&cvq->vring); 1945 1946 cvq->completed_desc++; 1947 queue_work(mvdev->wq, &wqent->work); 1948 break; 1949 } 1950 1951 out: 1952 up_write(&ndev->reslock); 1953 } 1954 1955 static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) 1956 { 1957 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 1958 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1959 struct mlx5_vdpa_virtqueue *mvq; 1960 1961 if (!is_index_valid(mvdev, idx)) 1962 return; 1963 1964 if (unlikely(is_ctrl_vq_idx(mvdev, idx))) { 1965 if (!mvdev->wq || !mvdev->cvq.ready) 1966 return; 1967 1968 queue_work(mvdev->wq, &ndev->cvq_ent.work); 1969 return; 1970 } 1971 1972 mvq = &ndev->vqs[idx]; 1973 if (unlikely(!mvq->ready)) 1974 return; 1975 1976 iowrite16(idx, ndev->mvdev.res.kick_addr); 1977 } 1978 1979 static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 1980 u64 driver_area, u64 device_area) 1981 { 1982 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 1983 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1984 struct mlx5_vdpa_virtqueue *mvq; 1985 1986 if (!is_index_valid(mvdev, idx)) 1987 return -EINVAL; 1988 1989 if (is_ctrl_vq_idx(mvdev, idx)) { 1990 mvdev->cvq.desc_addr = desc_area; 1991 mvdev->cvq.device_addr = device_area; 1992 mvdev->cvq.driver_addr = driver_area; 1993 return 0; 1994 } 1995 1996 mvq = &ndev->vqs[idx]; 1997 mvq->desc_addr = desc_area; 1998 mvq->device_addr = device_area; 1999 mvq->driver_addr = driver_area; 2000 return 0; 2001 } 2002 2003 static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 2004 { 2005 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2006 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2007 struct mlx5_vdpa_virtqueue *mvq; 2008 2009 if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2010 return; 2011 2012 mvq = &ndev->vqs[idx]; 2013 mvq->num_ent = num; 2014 } 2015 2016 static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 2017 { 2018 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2019 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2020 2021 ndev->event_cbs[idx] = *cb; 2022 if (is_ctrl_vq_idx(mvdev, idx)) 2023 mvdev->cvq.event_cb = *cb; 2024 } 2025 2026 static void mlx5_cvq_notify(struct vringh *vring) 2027 { 2028 struct mlx5_control_vq *cvq = container_of(vring, struct mlx5_control_vq, vring); 2029 2030 if (!cvq->event_cb.callback) 2031 return; 2032 2033 cvq->event_cb.callback(cvq->event_cb.private); 2034 } 2035 2036 static void set_cvq_ready(struct mlx5_vdpa_dev *mvdev, bool ready) 2037 { 2038 struct mlx5_control_vq *cvq = &mvdev->cvq; 2039 2040 cvq->ready = ready; 2041 if (!ready) 2042 return; 2043 2044 cvq->vring.notify = mlx5_cvq_notify; 2045 } 2046 2047 static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 2048 { 2049 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2050 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2051 struct mlx5_vdpa_virtqueue *mvq; 2052 int err; 2053 2054 if (!mvdev->actual_features) 2055 return; 2056 2057 if (!is_index_valid(mvdev, idx)) 2058 return; 2059 2060 if (is_ctrl_vq_idx(mvdev, idx)) { 2061 set_cvq_ready(mvdev, ready); 2062 return; 2063 } 2064 2065 mvq = &ndev->vqs[idx]; 2066 if (!ready) { 2067 suspend_vq(ndev, mvq); 2068 } else { 2069 err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 2070 if (err) { 2071 mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err); 2072 ready = false; 2073 } 2074 } 2075 2076 2077 mvq->ready = ready; 2078 } 2079 2080 static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx) 2081 { 2082 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2083 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2084 2085 if (!is_index_valid(mvdev, idx)) 2086 return false; 2087 2088 if (is_ctrl_vq_idx(mvdev, idx)) 2089 return mvdev->cvq.ready; 2090 2091 return ndev->vqs[idx].ready; 2092 } 2093 2094 static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx, 2095 const struct vdpa_vq_state *state) 2096 { 2097 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2098 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2099 struct mlx5_vdpa_virtqueue *mvq; 2100 2101 if (!is_index_valid(mvdev, idx)) 2102 return -EINVAL; 2103 2104 if (is_ctrl_vq_idx(mvdev, idx)) { 2105 mvdev->cvq.vring.last_avail_idx = state->split.avail_index; 2106 return 0; 2107 } 2108 2109 mvq = &ndev->vqs[idx]; 2110 if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) { 2111 mlx5_vdpa_warn(mvdev, "can't modify available index\n"); 2112 return -EINVAL; 2113 } 2114 2115 mvq->used_idx = state->split.avail_index; 2116 mvq->avail_idx = state->split.avail_index; 2117 return 0; 2118 } 2119 2120 static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 2121 { 2122 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2123 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2124 struct mlx5_vdpa_virtqueue *mvq; 2125 struct mlx5_virtq_attr attr; 2126 int err; 2127 2128 if (!is_index_valid(mvdev, idx)) 2129 return -EINVAL; 2130 2131 if (is_ctrl_vq_idx(mvdev, idx)) { 2132 state->split.avail_index = mvdev->cvq.vring.last_avail_idx; 2133 return 0; 2134 } 2135 2136 mvq = &ndev->vqs[idx]; 2137 /* If the virtq object was destroyed, use the value saved at 2138 * the last minute of suspend_vq. This caters for userspace 2139 * that cares about emulating the index after vq is stopped. 2140 */ 2141 if (!mvq->initialized) { 2142 /* Firmware returns a wrong value for the available index. 2143 * Since both values should be identical, we take the value of 2144 * used_idx which is reported correctly. 2145 */ 2146 state->split.avail_index = mvq->used_idx; 2147 return 0; 2148 } 2149 2150 err = query_virtqueue(ndev, mvq, &attr); 2151 if (err) { 2152 mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n"); 2153 return err; 2154 } 2155 state->split.avail_index = attr.used_index; 2156 return 0; 2157 } 2158 2159 static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) 2160 { 2161 return PAGE_SIZE; 2162 } 2163 2164 static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx) 2165 { 2166 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2167 2168 if (is_ctrl_vq_idx(mvdev, idx)) 2169 return MLX5_VDPA_CVQ_GROUP; 2170 2171 return MLX5_VDPA_DATAVQ_GROUP; 2172 } 2173 2174 enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9, 2175 MLX5_VIRTIO_NET_F_CSUM = 1 << 10, 2176 MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11, 2177 MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12, 2178 }; 2179 2180 static u64 mlx_to_vritio_features(u16 dev_features) 2181 { 2182 u64 result = 0; 2183 2184 if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM) 2185 result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM); 2186 if (dev_features & MLX5_VIRTIO_NET_F_CSUM) 2187 result |= BIT_ULL(VIRTIO_NET_F_CSUM); 2188 if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6) 2189 result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6); 2190 if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4) 2191 result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4); 2192 2193 return result; 2194 } 2195 2196 static u64 get_supported_features(struct mlx5_core_dev *mdev) 2197 { 2198 u64 mlx_vdpa_features = 0; 2199 u16 dev_features; 2200 2201 dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mdev, device_features_bits_mask); 2202 mlx_vdpa_features |= mlx_to_vritio_features(dev_features); 2203 if (MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_version_1_0)) 2204 mlx_vdpa_features |= BIT_ULL(VIRTIO_F_VERSION_1); 2205 mlx_vdpa_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM); 2206 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VQ); 2207 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR); 2208 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MQ); 2209 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_STATUS); 2210 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MTU); 2211 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VLAN); 2212 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MAC); 2213 2214 return mlx_vdpa_features; 2215 } 2216 2217 static u64 mlx5_vdpa_get_device_features(struct vdpa_device *vdev) 2218 { 2219 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2220 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2221 2222 print_features(mvdev, ndev->mvdev.mlx_features, false); 2223 return ndev->mvdev.mlx_features; 2224 } 2225 2226 static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features) 2227 { 2228 /* Minimum features to expect */ 2229 if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) 2230 return -EOPNOTSUPP; 2231 2232 /* Double check features combination sent down by the driver. 2233 * Fail invalid features due to absence of the depended feature. 2234 * 2235 * Per VIRTIO v1.1 specification, section 5.1.3.1 Feature bit 2236 * requirements: "VIRTIO_NET_F_MQ Requires VIRTIO_NET_F_CTRL_VQ". 2237 * By failing the invalid features sent down by untrusted drivers, 2238 * we're assured the assumption made upon is_index_valid() and 2239 * is_ctrl_vq_idx() will not be compromised. 2240 */ 2241 if ((features & (BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) == 2242 BIT_ULL(VIRTIO_NET_F_MQ)) 2243 return -EINVAL; 2244 2245 return 0; 2246 } 2247 2248 static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev) 2249 { 2250 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2251 int err; 2252 int i; 2253 2254 for (i = 0; i < mvdev->max_vqs; i++) { 2255 err = setup_vq(ndev, &ndev->vqs[i]); 2256 if (err) 2257 goto err_vq; 2258 } 2259 2260 return 0; 2261 2262 err_vq: 2263 for (--i; i >= 0; i--) 2264 teardown_vq(ndev, &ndev->vqs[i]); 2265 2266 return err; 2267 } 2268 2269 static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) 2270 { 2271 struct mlx5_vdpa_virtqueue *mvq; 2272 int i; 2273 2274 for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { 2275 mvq = &ndev->vqs[i]; 2276 if (!mvq->initialized) 2277 continue; 2278 2279 teardown_vq(ndev, mvq); 2280 } 2281 } 2282 2283 static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) 2284 { 2285 if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_CTRL_VQ)) { 2286 if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) { 2287 /* MQ supported. CVQ index is right above the last data virtqueue's */ 2288 mvdev->max_idx = mvdev->max_vqs; 2289 } else { 2290 /* Only CVQ supportted. data virtqueues occupy indices 0 and 1. 2291 * CVQ gets index 2 2292 */ 2293 mvdev->max_idx = 2; 2294 } 2295 } else { 2296 /* Two data virtqueues only: one for rx and one for tx */ 2297 mvdev->max_idx = 1; 2298 } 2299 } 2300 2301 static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) 2302 { 2303 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2304 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2305 int err; 2306 2307 print_features(mvdev, features, true); 2308 2309 err = verify_driver_features(mvdev, features); 2310 if (err) 2311 return err; 2312 2313 ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; 2314 if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)) 2315 ndev->rqt_size = mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs); 2316 else 2317 ndev->rqt_size = 1; 2318 2319 ndev->cur_num_vqs = 2 * ndev->rqt_size; 2320 2321 update_cvq_info(mvdev); 2322 return err; 2323 } 2324 2325 static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 2326 { 2327 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2328 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2329 2330 ndev->config_cb = *cb; 2331 } 2332 2333 #define MLX5_VDPA_MAX_VQ_ENTRIES 256 2334 static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev) 2335 { 2336 return MLX5_VDPA_MAX_VQ_ENTRIES; 2337 } 2338 2339 static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev) 2340 { 2341 return VIRTIO_ID_NET; 2342 } 2343 2344 static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev) 2345 { 2346 return PCI_VENDOR_ID_MELLANOX; 2347 } 2348 2349 static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev) 2350 { 2351 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2352 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2353 2354 print_status(mvdev, ndev->mvdev.status, false); 2355 return ndev->mvdev.status; 2356 } 2357 2358 static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 2359 { 2360 struct mlx5_vq_restore_info *ri = &mvq->ri; 2361 struct mlx5_virtq_attr attr = {}; 2362 int err; 2363 2364 if (mvq->initialized) { 2365 err = query_virtqueue(ndev, mvq, &attr); 2366 if (err) 2367 return err; 2368 } 2369 2370 ri->avail_index = attr.available_index; 2371 ri->used_index = attr.used_index; 2372 ri->ready = mvq->ready; 2373 ri->num_ent = mvq->num_ent; 2374 ri->desc_addr = mvq->desc_addr; 2375 ri->device_addr = mvq->device_addr; 2376 ri->driver_addr = mvq->driver_addr; 2377 ri->restore = true; 2378 return 0; 2379 } 2380 2381 static int save_channels_info(struct mlx5_vdpa_net *ndev) 2382 { 2383 int i; 2384 2385 for (i = 0; i < ndev->mvdev.max_vqs; i++) { 2386 memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); 2387 save_channel_info(ndev, &ndev->vqs[i]); 2388 } 2389 return 0; 2390 } 2391 2392 static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) 2393 { 2394 int i; 2395 2396 for (i = 0; i < ndev->mvdev.max_vqs; i++) 2397 memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 2398 } 2399 2400 static void restore_channels_info(struct mlx5_vdpa_net *ndev) 2401 { 2402 struct mlx5_vdpa_virtqueue *mvq; 2403 struct mlx5_vq_restore_info *ri; 2404 int i; 2405 2406 mlx5_clear_vqs(ndev); 2407 init_mvqs(ndev); 2408 for (i = 0; i < ndev->mvdev.max_vqs; i++) { 2409 mvq = &ndev->vqs[i]; 2410 ri = &mvq->ri; 2411 if (!ri->restore) 2412 continue; 2413 2414 mvq->avail_idx = ri->avail_index; 2415 mvq->used_idx = ri->used_index; 2416 mvq->ready = ri->ready; 2417 mvq->num_ent = ri->num_ent; 2418 mvq->desc_addr = ri->desc_addr; 2419 mvq->device_addr = ri->device_addr; 2420 mvq->driver_addr = ri->driver_addr; 2421 } 2422 } 2423 2424 static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, 2425 struct vhost_iotlb *iotlb, unsigned int asid) 2426 { 2427 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2428 int err; 2429 2430 suspend_vqs(ndev); 2431 err = save_channels_info(ndev); 2432 if (err) 2433 goto err_mr; 2434 2435 teardown_driver(ndev); 2436 mlx5_vdpa_destroy_mr(mvdev); 2437 err = mlx5_vdpa_create_mr(mvdev, iotlb, asid); 2438 if (err) 2439 goto err_mr; 2440 2441 if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 2442 goto err_mr; 2443 2444 restore_channels_info(ndev); 2445 err = setup_driver(mvdev); 2446 if (err) 2447 goto err_setup; 2448 2449 return 0; 2450 2451 err_setup: 2452 mlx5_vdpa_destroy_mr(mvdev); 2453 err_mr: 2454 return err; 2455 } 2456 2457 /* reslock must be held for this function */ 2458 static int setup_driver(struct mlx5_vdpa_dev *mvdev) 2459 { 2460 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2461 int err; 2462 2463 WARN_ON(!rwsem_is_locked(&ndev->reslock)); 2464 2465 if (ndev->setup) { 2466 mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n"); 2467 err = 0; 2468 goto out; 2469 } 2470 err = setup_virtqueues(mvdev); 2471 if (err) { 2472 mlx5_vdpa_warn(mvdev, "setup_virtqueues\n"); 2473 goto out; 2474 } 2475 2476 err = create_rqt(ndev); 2477 if (err) { 2478 mlx5_vdpa_warn(mvdev, "create_rqt\n"); 2479 goto err_rqt; 2480 } 2481 2482 err = create_tir(ndev); 2483 if (err) { 2484 mlx5_vdpa_warn(mvdev, "create_tir\n"); 2485 goto err_tir; 2486 } 2487 2488 err = setup_steering(ndev); 2489 if (err) { 2490 mlx5_vdpa_warn(mvdev, "setup_steering\n"); 2491 goto err_fwd; 2492 } 2493 ndev->setup = true; 2494 2495 return 0; 2496 2497 err_fwd: 2498 destroy_tir(ndev); 2499 err_tir: 2500 destroy_rqt(ndev); 2501 err_rqt: 2502 teardown_virtqueues(ndev); 2503 out: 2504 return err; 2505 } 2506 2507 /* reslock must be held for this function */ 2508 static void teardown_driver(struct mlx5_vdpa_net *ndev) 2509 { 2510 2511 WARN_ON(!rwsem_is_locked(&ndev->reslock)); 2512 2513 if (!ndev->setup) 2514 return; 2515 2516 teardown_steering(ndev); 2517 destroy_tir(ndev); 2518 destroy_rqt(ndev); 2519 teardown_virtqueues(ndev); 2520 ndev->setup = false; 2521 } 2522 2523 static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) 2524 { 2525 int i; 2526 2527 for (i = 0; i < ndev->mvdev.max_vqs; i++) 2528 ndev->vqs[i].ready = false; 2529 2530 ndev->mvdev.cvq.ready = false; 2531 } 2532 2533 static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev) 2534 { 2535 struct mlx5_control_vq *cvq = &mvdev->cvq; 2536 int err = 0; 2537 2538 if (mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) 2539 err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features, 2540 MLX5_CVQ_MAX_ENT, false, 2541 (struct vring_desc *)(uintptr_t)cvq->desc_addr, 2542 (struct vring_avail *)(uintptr_t)cvq->driver_addr, 2543 (struct vring_used *)(uintptr_t)cvq->device_addr); 2544 2545 return err; 2546 } 2547 2548 static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) 2549 { 2550 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2551 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2552 int err; 2553 2554 print_status(mvdev, status, true); 2555 2556 down_write(&ndev->reslock); 2557 2558 if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { 2559 if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 2560 err = setup_cvq_vring(mvdev); 2561 if (err) { 2562 mlx5_vdpa_warn(mvdev, "failed to setup control VQ vring\n"); 2563 goto err_setup; 2564 } 2565 err = setup_driver(mvdev); 2566 if (err) { 2567 mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); 2568 goto err_setup; 2569 } 2570 } else { 2571 mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); 2572 goto err_clear; 2573 } 2574 } 2575 2576 ndev->mvdev.status = status; 2577 up_write(&ndev->reslock); 2578 return; 2579 2580 err_setup: 2581 mlx5_vdpa_destroy_mr(&ndev->mvdev); 2582 ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; 2583 err_clear: 2584 up_write(&ndev->reslock); 2585 } 2586 2587 static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev) 2588 { 2589 int i; 2590 2591 /* default mapping all groups are mapped to asid 0 */ 2592 for (i = 0; i < MLX5_VDPA_NUMVQ_GROUPS; i++) 2593 mvdev->group2asid[i] = 0; 2594 } 2595 2596 static int mlx5_vdpa_reset(struct vdpa_device *vdev) 2597 { 2598 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2599 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2600 2601 print_status(mvdev, 0, true); 2602 mlx5_vdpa_info(mvdev, "performing device reset\n"); 2603 2604 down_write(&ndev->reslock); 2605 teardown_driver(ndev); 2606 clear_vqs_ready(ndev); 2607 mlx5_vdpa_destroy_mr(&ndev->mvdev); 2608 ndev->mvdev.status = 0; 2609 ndev->cur_num_vqs = 0; 2610 ndev->mvdev.cvq.received_desc = 0; 2611 ndev->mvdev.cvq.completed_desc = 0; 2612 memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1)); 2613 ndev->mvdev.actual_features = 0; 2614 init_group_to_asid_map(mvdev); 2615 ++mvdev->generation; 2616 2617 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 2618 if (mlx5_vdpa_create_mr(mvdev, NULL, 0)) 2619 mlx5_vdpa_warn(mvdev, "create MR failed\n"); 2620 } 2621 up_write(&ndev->reslock); 2622 2623 return 0; 2624 } 2625 2626 static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev) 2627 { 2628 return sizeof(struct virtio_net_config); 2629 } 2630 2631 static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, 2632 unsigned int len) 2633 { 2634 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2635 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2636 2637 if (offset + len <= sizeof(struct virtio_net_config)) 2638 memcpy(buf, (u8 *)&ndev->config + offset, len); 2639 } 2640 2641 static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf, 2642 unsigned int len) 2643 { 2644 /* not supported */ 2645 } 2646 2647 static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) 2648 { 2649 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2650 2651 return mvdev->generation; 2652 } 2653 2654 static int set_map_data(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb, 2655 unsigned int asid) 2656 { 2657 bool change_map; 2658 int err; 2659 2660 err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map, asid); 2661 if (err) { 2662 mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); 2663 return err; 2664 } 2665 2666 if (change_map) 2667 err = mlx5_vdpa_change_map(mvdev, iotlb, asid); 2668 2669 return err; 2670 } 2671 2672 static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid, 2673 struct vhost_iotlb *iotlb) 2674 { 2675 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2676 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2677 int err = -EINVAL; 2678 2679 down_write(&ndev->reslock); 2680 err = set_map_data(mvdev, iotlb, asid); 2681 up_write(&ndev->reslock); 2682 return err; 2683 } 2684 2685 static struct device *mlx5_get_vq_dma_dev(struct vdpa_device *vdev, u16 idx) 2686 { 2687 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2688 2689 if (is_ctrl_vq_idx(mvdev, idx)) 2690 return &vdev->dev; 2691 2692 return mvdev->vdev.dma_dev; 2693 } 2694 2695 static void mlx5_vdpa_free(struct vdpa_device *vdev) 2696 { 2697 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2698 struct mlx5_core_dev *pfmdev; 2699 struct mlx5_vdpa_net *ndev; 2700 2701 ndev = to_mlx5_vdpa_ndev(mvdev); 2702 2703 free_resources(ndev); 2704 mlx5_vdpa_destroy_mr(mvdev); 2705 if (!is_zero_ether_addr(ndev->config.mac)) { 2706 pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 2707 mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); 2708 } 2709 mlx5_vdpa_free_resources(&ndev->mvdev); 2710 kfree(ndev->event_cbs); 2711 kfree(ndev->vqs); 2712 } 2713 2714 static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx) 2715 { 2716 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2717 struct vdpa_notification_area ret = {}; 2718 struct mlx5_vdpa_net *ndev; 2719 phys_addr_t addr; 2720 2721 if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2722 return ret; 2723 2724 /* If SF BAR size is smaller than PAGE_SIZE, do not use direct 2725 * notification to avoid the risk of mapping pages that contain BAR of more 2726 * than one SF 2727 */ 2728 if (MLX5_CAP_GEN(mvdev->mdev, log_min_sf_size) + 12 < PAGE_SHIFT) 2729 return ret; 2730 2731 ndev = to_mlx5_vdpa_ndev(mvdev); 2732 addr = (phys_addr_t)ndev->mvdev.res.phys_kick_addr; 2733 ret.addr = addr; 2734 ret.size = PAGE_SIZE; 2735 return ret; 2736 } 2737 2738 static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) 2739 { 2740 return -EOPNOTSUPP; 2741 } 2742 2743 static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) 2744 { 2745 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2746 2747 return mvdev->actual_features; 2748 } 2749 2750 static int counter_set_query(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 2751 u64 *received_desc, u64 *completed_desc) 2752 { 2753 u32 in[MLX5_ST_SZ_DW(query_virtio_q_counters_in)] = {}; 2754 u32 out[MLX5_ST_SZ_DW(query_virtio_q_counters_out)] = {}; 2755 void *cmd_hdr; 2756 void *ctx; 2757 int err; 2758 2759 if (!counters_supported(&ndev->mvdev)) 2760 return -EOPNOTSUPP; 2761 2762 if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 2763 return -EAGAIN; 2764 2765 cmd_hdr = MLX5_ADDR_OF(query_virtio_q_counters_in, in, hdr); 2766 2767 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 2768 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 2769 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 2770 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->counter_set_id); 2771 2772 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 2773 if (err) 2774 return err; 2775 2776 ctx = MLX5_ADDR_OF(query_virtio_q_counters_out, out, counters); 2777 *received_desc = MLX5_GET64(virtio_q_counters, ctx, received_desc); 2778 *completed_desc = MLX5_GET64(virtio_q_counters, ctx, completed_desc); 2779 return 0; 2780 } 2781 2782 static int mlx5_vdpa_get_vendor_vq_stats(struct vdpa_device *vdev, u16 idx, 2783 struct sk_buff *msg, 2784 struct netlink_ext_ack *extack) 2785 { 2786 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2787 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2788 struct mlx5_vdpa_virtqueue *mvq; 2789 struct mlx5_control_vq *cvq; 2790 u64 received_desc; 2791 u64 completed_desc; 2792 int err = 0; 2793 2794 down_read(&ndev->reslock); 2795 if (!is_index_valid(mvdev, idx)) { 2796 NL_SET_ERR_MSG_MOD(extack, "virtqueue index is not valid"); 2797 err = -EINVAL; 2798 goto out_err; 2799 } 2800 2801 if (idx == ctrl_vq_idx(mvdev)) { 2802 cvq = &mvdev->cvq; 2803 received_desc = cvq->received_desc; 2804 completed_desc = cvq->completed_desc; 2805 goto out; 2806 } 2807 2808 mvq = &ndev->vqs[idx]; 2809 err = counter_set_query(ndev, mvq, &received_desc, &completed_desc); 2810 if (err) { 2811 NL_SET_ERR_MSG_MOD(extack, "failed to query hardware"); 2812 goto out_err; 2813 } 2814 2815 out: 2816 err = -EMSGSIZE; 2817 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "received_desc")) 2818 goto out_err; 2819 2820 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, received_desc, 2821 VDPA_ATTR_PAD)) 2822 goto out_err; 2823 2824 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "completed_desc")) 2825 goto out_err; 2826 2827 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, completed_desc, 2828 VDPA_ATTR_PAD)) 2829 goto out_err; 2830 2831 err = 0; 2832 out_err: 2833 up_read(&ndev->reslock); 2834 return err; 2835 } 2836 2837 static void mlx5_vdpa_cvq_suspend(struct mlx5_vdpa_dev *mvdev) 2838 { 2839 struct mlx5_control_vq *cvq; 2840 2841 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 2842 return; 2843 2844 cvq = &mvdev->cvq; 2845 cvq->ready = false; 2846 } 2847 2848 static int mlx5_vdpa_suspend(struct vdpa_device *vdev) 2849 { 2850 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2851 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2852 struct mlx5_vdpa_virtqueue *mvq; 2853 int i; 2854 2855 down_write(&ndev->reslock); 2856 ndev->nb_registered = false; 2857 mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 2858 flush_workqueue(ndev->mvdev.wq); 2859 for (i = 0; i < ndev->cur_num_vqs; i++) { 2860 mvq = &ndev->vqs[i]; 2861 suspend_vq(ndev, mvq); 2862 } 2863 mlx5_vdpa_cvq_suspend(mvdev); 2864 up_write(&ndev->reslock); 2865 return 0; 2866 } 2867 2868 static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group, 2869 unsigned int asid) 2870 { 2871 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2872 2873 if (group >= MLX5_VDPA_NUMVQ_GROUPS) 2874 return -EINVAL; 2875 2876 mvdev->group2asid[group] = asid; 2877 return 0; 2878 } 2879 2880 static const struct vdpa_config_ops mlx5_vdpa_ops = { 2881 .set_vq_address = mlx5_vdpa_set_vq_address, 2882 .set_vq_num = mlx5_vdpa_set_vq_num, 2883 .kick_vq = mlx5_vdpa_kick_vq, 2884 .set_vq_cb = mlx5_vdpa_set_vq_cb, 2885 .set_vq_ready = mlx5_vdpa_set_vq_ready, 2886 .get_vq_ready = mlx5_vdpa_get_vq_ready, 2887 .set_vq_state = mlx5_vdpa_set_vq_state, 2888 .get_vq_state = mlx5_vdpa_get_vq_state, 2889 .get_vendor_vq_stats = mlx5_vdpa_get_vendor_vq_stats, 2890 .get_vq_notification = mlx5_get_vq_notification, 2891 .get_vq_irq = mlx5_get_vq_irq, 2892 .get_vq_align = mlx5_vdpa_get_vq_align, 2893 .get_vq_group = mlx5_vdpa_get_vq_group, 2894 .get_device_features = mlx5_vdpa_get_device_features, 2895 .set_driver_features = mlx5_vdpa_set_driver_features, 2896 .get_driver_features = mlx5_vdpa_get_driver_features, 2897 .set_config_cb = mlx5_vdpa_set_config_cb, 2898 .get_vq_num_max = mlx5_vdpa_get_vq_num_max, 2899 .get_device_id = mlx5_vdpa_get_device_id, 2900 .get_vendor_id = mlx5_vdpa_get_vendor_id, 2901 .get_status = mlx5_vdpa_get_status, 2902 .set_status = mlx5_vdpa_set_status, 2903 .reset = mlx5_vdpa_reset, 2904 .get_config_size = mlx5_vdpa_get_config_size, 2905 .get_config = mlx5_vdpa_get_config, 2906 .set_config = mlx5_vdpa_set_config, 2907 .get_generation = mlx5_vdpa_get_generation, 2908 .set_map = mlx5_vdpa_set_map, 2909 .set_group_asid = mlx5_set_group_asid, 2910 .get_vq_dma_dev = mlx5_get_vq_dma_dev, 2911 .free = mlx5_vdpa_free, 2912 .suspend = mlx5_vdpa_suspend, 2913 }; 2914 2915 static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) 2916 { 2917 u16 hw_mtu; 2918 int err; 2919 2920 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2921 if (err) 2922 return err; 2923 2924 *mtu = hw_mtu - MLX5V_ETH_HARD_MTU; 2925 return 0; 2926 } 2927 2928 static int alloc_resources(struct mlx5_vdpa_net *ndev) 2929 { 2930 struct mlx5_vdpa_net_resources *res = &ndev->res; 2931 int err; 2932 2933 if (res->valid) { 2934 mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); 2935 return -EEXIST; 2936 } 2937 2938 err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); 2939 if (err) 2940 return err; 2941 2942 err = create_tis(ndev); 2943 if (err) 2944 goto err_tis; 2945 2946 res->valid = true; 2947 2948 return 0; 2949 2950 err_tis: 2951 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 2952 return err; 2953 } 2954 2955 static void free_resources(struct mlx5_vdpa_net *ndev) 2956 { 2957 struct mlx5_vdpa_net_resources *res = &ndev->res; 2958 2959 if (!res->valid) 2960 return; 2961 2962 destroy_tis(ndev); 2963 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 2964 res->valid = false; 2965 } 2966 2967 static void init_mvqs(struct mlx5_vdpa_net *ndev) 2968 { 2969 struct mlx5_vdpa_virtqueue *mvq; 2970 int i; 2971 2972 for (i = 0; i < ndev->mvdev.max_vqs; ++i) { 2973 mvq = &ndev->vqs[i]; 2974 memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 2975 mvq->index = i; 2976 mvq->ndev = ndev; 2977 mvq->fwqp.fw = true; 2978 mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; 2979 } 2980 for (; i < ndev->mvdev.max_vqs; i++) { 2981 mvq = &ndev->vqs[i]; 2982 memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 2983 mvq->index = i; 2984 mvq->ndev = ndev; 2985 } 2986 } 2987 2988 struct mlx5_vdpa_mgmtdev { 2989 struct vdpa_mgmt_dev mgtdev; 2990 struct mlx5_adev *madev; 2991 struct mlx5_vdpa_net *ndev; 2992 }; 2993 2994 static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) 2995 { 2996 u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; 2997 u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; 2998 int err; 2999 3000 MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE); 3001 MLX5_SET(query_vport_state_in, in, op_mod, opmod); 3002 MLX5_SET(query_vport_state_in, in, vport_number, vport); 3003 if (vport) 3004 MLX5_SET(query_vport_state_in, in, other_vport, 1); 3005 3006 err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out); 3007 if (err) 3008 return 0; 3009 3010 return MLX5_GET(query_vport_state_out, out, state); 3011 } 3012 3013 static bool get_link_state(struct mlx5_vdpa_dev *mvdev) 3014 { 3015 if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) == 3016 VPORT_STATE_UP) 3017 return true; 3018 3019 return false; 3020 } 3021 3022 static void update_carrier(struct work_struct *work) 3023 { 3024 struct mlx5_vdpa_wq_ent *wqent; 3025 struct mlx5_vdpa_dev *mvdev; 3026 struct mlx5_vdpa_net *ndev; 3027 3028 wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 3029 mvdev = wqent->mvdev; 3030 ndev = to_mlx5_vdpa_ndev(mvdev); 3031 if (get_link_state(mvdev)) 3032 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 3033 else 3034 ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 3035 3036 if (ndev->nb_registered && ndev->config_cb.callback) 3037 ndev->config_cb.callback(ndev->config_cb.private); 3038 3039 kfree(wqent); 3040 } 3041 3042 static int event_handler(struct notifier_block *nb, unsigned long event, void *param) 3043 { 3044 struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb); 3045 struct mlx5_eqe *eqe = param; 3046 int ret = NOTIFY_DONE; 3047 struct mlx5_vdpa_wq_ent *wqent; 3048 3049 if (event == MLX5_EVENT_TYPE_PORT_CHANGE) { 3050 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_STATUS))) 3051 return NOTIFY_DONE; 3052 switch (eqe->sub_type) { 3053 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 3054 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 3055 wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); 3056 if (!wqent) 3057 return NOTIFY_DONE; 3058 3059 wqent->mvdev = &ndev->mvdev; 3060 INIT_WORK(&wqent->work, update_carrier); 3061 queue_work(ndev->mvdev.wq, &wqent->work); 3062 ret = NOTIFY_OK; 3063 break; 3064 default: 3065 return NOTIFY_DONE; 3066 } 3067 return ret; 3068 } 3069 return ret; 3070 } 3071 3072 static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu) 3073 { 3074 int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); 3075 void *in; 3076 int err; 3077 3078 in = kvzalloc(inlen, GFP_KERNEL); 3079 if (!in) 3080 return -ENOMEM; 3081 3082 MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1); 3083 MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu, 3084 mtu + MLX5V_ETH_HARD_MTU); 3085 MLX5_SET(modify_nic_vport_context_in, in, opcode, 3086 MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); 3087 3088 err = mlx5_cmd_exec_in(mdev, modify_nic_vport_context, in); 3089 3090 kvfree(in); 3091 return err; 3092 } 3093 3094 static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, 3095 const struct vdpa_dev_set_config *add_config) 3096 { 3097 struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 3098 struct virtio_net_config *config; 3099 struct mlx5_core_dev *pfmdev; 3100 struct mlx5_vdpa_dev *mvdev; 3101 struct mlx5_vdpa_net *ndev; 3102 struct mlx5_core_dev *mdev; 3103 u64 device_features; 3104 u32 max_vqs; 3105 u16 mtu; 3106 int err; 3107 3108 if (mgtdev->ndev) 3109 return -ENOSPC; 3110 3111 mdev = mgtdev->madev->mdev; 3112 device_features = mgtdev->mgtdev.supported_features; 3113 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { 3114 if (add_config->device_features & ~device_features) { 3115 dev_warn(mdev->device, 3116 "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n", 3117 add_config->device_features, device_features); 3118 return -EINVAL; 3119 } 3120 device_features &= add_config->device_features; 3121 } 3122 if (!(device_features & BIT_ULL(VIRTIO_F_VERSION_1) && 3123 device_features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) { 3124 dev_warn(mdev->device, 3125 "Must provision minimum features 0x%llx for this device", 3126 BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)); 3127 return -EOPNOTSUPP; 3128 } 3129 3130 if (!(MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_queue_type) & 3131 MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)) { 3132 dev_warn(mdev->device, "missing support for split virtqueues\n"); 3133 return -EOPNOTSUPP; 3134 } 3135 3136 max_vqs = min_t(int, MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues), 3137 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size)); 3138 if (max_vqs < 2) { 3139 dev_warn(mdev->device, 3140 "%d virtqueues are supported. At least 2 are required\n", 3141 max_vqs); 3142 return -EAGAIN; 3143 } 3144 3145 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP)) { 3146 if (add_config->net.max_vq_pairs > max_vqs / 2) 3147 return -EINVAL; 3148 max_vqs = min_t(u32, max_vqs, 2 * add_config->net.max_vq_pairs); 3149 } else { 3150 max_vqs = 2; 3151 } 3152 3153 ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, 3154 MLX5_VDPA_NUMVQ_GROUPS, MLX5_VDPA_NUM_AS, name, false); 3155 if (IS_ERR(ndev)) 3156 return PTR_ERR(ndev); 3157 3158 ndev->mvdev.max_vqs = max_vqs; 3159 mvdev = &ndev->mvdev; 3160 mvdev->mdev = mdev; 3161 3162 ndev->vqs = kcalloc(max_vqs, sizeof(*ndev->vqs), GFP_KERNEL); 3163 ndev->event_cbs = kcalloc(max_vqs + 1, sizeof(*ndev->event_cbs), GFP_KERNEL); 3164 if (!ndev->vqs || !ndev->event_cbs) { 3165 err = -ENOMEM; 3166 goto err_alloc; 3167 } 3168 3169 init_mvqs(ndev); 3170 init_rwsem(&ndev->reslock); 3171 config = &ndev->config; 3172 3173 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU)) { 3174 err = config_func_mtu(mdev, add_config->net.mtu); 3175 if (err) 3176 goto err_alloc; 3177 } 3178 3179 if (device_features & BIT_ULL(VIRTIO_NET_F_MTU)) { 3180 err = query_mtu(mdev, &mtu); 3181 if (err) 3182 goto err_alloc; 3183 3184 ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, mtu); 3185 } 3186 3187 if (device_features & BIT_ULL(VIRTIO_NET_F_STATUS)) { 3188 if (get_link_state(mvdev)) 3189 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 3190 else 3191 ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 3192 } 3193 3194 if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) { 3195 memcpy(ndev->config.mac, add_config->net.mac, ETH_ALEN); 3196 /* No bother setting mac address in config if not going to provision _F_MAC */ 3197 } else if ((add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) == 0 || 3198 device_features & BIT_ULL(VIRTIO_NET_F_MAC)) { 3199 err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac); 3200 if (err) 3201 goto err_alloc; 3202 } 3203 3204 if (!is_zero_ether_addr(config->mac)) { 3205 pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev)); 3206 err = mlx5_mpfs_add_mac(pfmdev, config->mac); 3207 if (err) 3208 goto err_alloc; 3209 } else if ((add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) == 0) { 3210 /* 3211 * We used to clear _F_MAC feature bit if seeing 3212 * zero mac address when device features are not 3213 * specifically provisioned. Keep the behaviour 3214 * so old scripts do not break. 3215 */ 3216 device_features &= ~BIT_ULL(VIRTIO_NET_F_MAC); 3217 } else if (device_features & BIT_ULL(VIRTIO_NET_F_MAC)) { 3218 /* Don't provision zero mac address for _F_MAC */ 3219 mlx5_vdpa_warn(&ndev->mvdev, 3220 "No mac address provisioned?\n"); 3221 err = -EINVAL; 3222 goto err_alloc; 3223 } 3224 3225 if (device_features & BIT_ULL(VIRTIO_NET_F_MQ)) 3226 config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, max_vqs / 2); 3227 3228 ndev->mvdev.mlx_features = device_features; 3229 mvdev->vdev.dma_dev = &mdev->pdev->dev; 3230 err = mlx5_vdpa_alloc_resources(&ndev->mvdev); 3231 if (err) 3232 goto err_mpfs; 3233 3234 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 3235 err = mlx5_vdpa_create_mr(mvdev, NULL, 0); 3236 if (err) 3237 goto err_res; 3238 } 3239 3240 err = alloc_resources(ndev); 3241 if (err) 3242 goto err_mr; 3243 3244 ndev->cvq_ent.mvdev = mvdev; 3245 INIT_WORK(&ndev->cvq_ent.work, mlx5_cvq_kick_handler); 3246 mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_wq"); 3247 if (!mvdev->wq) { 3248 err = -ENOMEM; 3249 goto err_res2; 3250 } 3251 3252 ndev->nb.notifier_call = event_handler; 3253 mlx5_notifier_register(mdev, &ndev->nb); 3254 ndev->nb_registered = true; 3255 mvdev->vdev.mdev = &mgtdev->mgtdev; 3256 err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1); 3257 if (err) 3258 goto err_reg; 3259 3260 mlx5_vdpa_add_debugfs(ndev); 3261 mgtdev->ndev = ndev; 3262 return 0; 3263 3264 err_reg: 3265 destroy_workqueue(mvdev->wq); 3266 err_res2: 3267 free_resources(ndev); 3268 err_mr: 3269 mlx5_vdpa_destroy_mr(mvdev); 3270 err_res: 3271 mlx5_vdpa_free_resources(&ndev->mvdev); 3272 err_mpfs: 3273 if (!is_zero_ether_addr(config->mac)) 3274 mlx5_mpfs_del_mac(pfmdev, config->mac); 3275 err_alloc: 3276 put_device(&mvdev->vdev.dev); 3277 return err; 3278 } 3279 3280 static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev) 3281 { 3282 struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 3283 struct mlx5_vdpa_dev *mvdev = to_mvdev(dev); 3284 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 3285 struct workqueue_struct *wq; 3286 3287 mlx5_vdpa_remove_debugfs(ndev->debugfs); 3288 ndev->debugfs = NULL; 3289 if (ndev->nb_registered) { 3290 ndev->nb_registered = false; 3291 mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 3292 } 3293 wq = mvdev->wq; 3294 mvdev->wq = NULL; 3295 destroy_workqueue(wq); 3296 _vdpa_unregister_device(dev); 3297 mgtdev->ndev = NULL; 3298 } 3299 3300 static const struct vdpa_mgmtdev_ops mdev_ops = { 3301 .dev_add = mlx5_vdpa_dev_add, 3302 .dev_del = mlx5_vdpa_dev_del, 3303 }; 3304 3305 static struct virtio_device_id id_table[] = { 3306 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 3307 { 0 }, 3308 }; 3309 3310 static int mlx5v_probe(struct auxiliary_device *adev, 3311 const struct auxiliary_device_id *id) 3312 3313 { 3314 struct mlx5_adev *madev = container_of(adev, struct mlx5_adev, adev); 3315 struct mlx5_core_dev *mdev = madev->mdev; 3316 struct mlx5_vdpa_mgmtdev *mgtdev; 3317 int err; 3318 3319 mgtdev = kzalloc(sizeof(*mgtdev), GFP_KERNEL); 3320 if (!mgtdev) 3321 return -ENOMEM; 3322 3323 mgtdev->mgtdev.ops = &mdev_ops; 3324 mgtdev->mgtdev.device = mdev->device; 3325 mgtdev->mgtdev.id_table = id_table; 3326 mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | 3327 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP) | 3328 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU) | 3329 BIT_ULL(VDPA_ATTR_DEV_FEATURES); 3330 mgtdev->mgtdev.max_supported_vqs = 3331 MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1; 3332 mgtdev->mgtdev.supported_features = get_supported_features(mdev); 3333 mgtdev->madev = madev; 3334 3335 err = vdpa_mgmtdev_register(&mgtdev->mgtdev); 3336 if (err) 3337 goto reg_err; 3338 3339 auxiliary_set_drvdata(adev, mgtdev); 3340 3341 return 0; 3342 3343 reg_err: 3344 kfree(mgtdev); 3345 return err; 3346 } 3347 3348 static void mlx5v_remove(struct auxiliary_device *adev) 3349 { 3350 struct mlx5_vdpa_mgmtdev *mgtdev; 3351 3352 mgtdev = auxiliary_get_drvdata(adev); 3353 vdpa_mgmtdev_unregister(&mgtdev->mgtdev); 3354 kfree(mgtdev); 3355 } 3356 3357 static const struct auxiliary_device_id mlx5v_id_table[] = { 3358 { .name = MLX5_ADEV_NAME ".vnet", }, 3359 {}, 3360 }; 3361 3362 MODULE_DEVICE_TABLE(auxiliary, mlx5v_id_table); 3363 3364 static struct auxiliary_driver mlx5v_driver = { 3365 .name = "vnet", 3366 .probe = mlx5v_probe, 3367 .remove = mlx5v_remove, 3368 .id_table = mlx5v_id_table, 3369 }; 3370 3371 module_auxiliary_driver(mlx5v_driver); 3372