1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <rdma/ib_user_verbs.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/uverbs_types.h> 9 #include <rdma/uverbs_ioctl.h> 10 #include <rdma/mlx5_user_ioctl_cmds.h> 11 #include <rdma/mlx5_user_ioctl_verbs.h> 12 #include <rdma/ib_umem.h> 13 #include <rdma/uverbs_std_types.h> 14 #include <linux/mlx5/driver.h> 15 #include <linux/mlx5/fs.h> 16 #include "mlx5_ib.h" 17 18 #define UVERBS_MODULE_NAME mlx5_ib 19 #include <rdma/uverbs_named_ioctl.h> 20 21 enum devx_obj_flags { 22 DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0, 23 DEVX_OBJ_FLAGS_DCT = 1 << 1, 24 }; 25 26 struct devx_async_data { 27 struct mlx5_ib_dev *mdev; 28 struct list_head list; 29 struct ib_uobject *fd_uobj; 30 struct mlx5_async_work cb_work; 31 u16 cmd_out_len; 32 /* must be last field in this structure */ 33 struct mlx5_ib_uapi_devx_async_cmd_hdr hdr; 34 }; 35 36 #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) 37 struct devx_obj { 38 struct mlx5_core_dev *mdev; 39 u64 obj_id; 40 u32 dinlen; /* destroy inbox length */ 41 u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW]; 42 u32 flags; 43 union { 44 struct mlx5_ib_devx_mr devx_mr; 45 struct mlx5_core_dct core_dct; 46 }; 47 }; 48 49 struct devx_umem { 50 struct mlx5_core_dev *mdev; 51 struct ib_umem *umem; 52 u32 page_offset; 53 int page_shift; 54 int ncont; 55 u32 dinlen; 56 u32 dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)]; 57 }; 58 59 struct devx_umem_reg_cmd { 60 void *in; 61 u32 inlen; 62 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 63 }; 64 65 static struct mlx5_ib_ucontext * 66 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs) 67 { 68 return to_mucontext(ib_uverbs_get_ucontext(attrs)); 69 } 70 71 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user) 72 { 73 u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0}; 74 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0}; 75 void *uctx; 76 int err; 77 u16 uid; 78 u32 cap = 0; 79 80 /* 0 means not supported */ 81 if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx)) 82 return -EINVAL; 83 84 uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx); 85 if (is_user && capable(CAP_NET_RAW) && 86 (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX)) 87 cap |= MLX5_UCTX_CAP_RAW_TX; 88 if (is_user && capable(CAP_SYS_RAWIO) && 89 (MLX5_CAP_GEN(dev->mdev, uctx_cap) & 90 MLX5_UCTX_CAP_INTERNAL_DEV_RES)) 91 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES; 92 93 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX); 94 MLX5_SET(uctx, uctx, cap, cap); 95 96 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 97 if (err) 98 return err; 99 100 uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 101 return uid; 102 } 103 104 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) 105 { 106 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0}; 107 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0}; 108 109 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX); 110 MLX5_SET(destroy_uctx_in, in, uid, uid); 111 112 mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 113 } 114 115 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type) 116 { 117 struct devx_obj *devx_obj = obj; 118 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 119 120 switch (opcode) { 121 case MLX5_CMD_OP_DESTROY_TIR: 122 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 123 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 124 obj_id); 125 return true; 126 127 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 128 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 129 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 130 table_id); 131 return true; 132 default: 133 return false; 134 } 135 } 136 137 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id) 138 { 139 struct devx_obj *devx_obj = obj; 140 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 141 142 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 143 *counter_id = MLX5_GET(dealloc_flow_counter_in, 144 devx_obj->dinbox, 145 flow_counter_id); 146 return true; 147 } 148 149 return false; 150 } 151 152 /* 153 * As the obj_id in the firmware is not globally unique the object type 154 * must be considered upon checking for a valid object id. 155 * For that the opcode of the creator command is encoded as part of the obj_id. 156 */ 157 static u64 get_enc_obj_id(u32 opcode, u32 obj_id) 158 { 159 return ((u64)opcode << 32) | obj_id; 160 } 161 162 static u64 devx_get_obj_id(const void *in) 163 { 164 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 165 u64 obj_id; 166 167 switch (opcode) { 168 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 169 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 170 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT | 171 MLX5_GET(general_obj_in_cmd_hdr, in, 172 obj_type) << 16, 173 MLX5_GET(general_obj_in_cmd_hdr, in, 174 obj_id)); 175 break; 176 case MLX5_CMD_OP_QUERY_MKEY: 177 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY, 178 MLX5_GET(query_mkey_in, in, 179 mkey_index)); 180 break; 181 case MLX5_CMD_OP_QUERY_CQ: 182 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ, 183 MLX5_GET(query_cq_in, in, cqn)); 184 break; 185 case MLX5_CMD_OP_MODIFY_CQ: 186 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ, 187 MLX5_GET(modify_cq_in, in, cqn)); 188 break; 189 case MLX5_CMD_OP_QUERY_SQ: 190 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ, 191 MLX5_GET(query_sq_in, in, sqn)); 192 break; 193 case MLX5_CMD_OP_MODIFY_SQ: 194 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ, 195 MLX5_GET(modify_sq_in, in, sqn)); 196 break; 197 case MLX5_CMD_OP_QUERY_RQ: 198 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 199 MLX5_GET(query_rq_in, in, rqn)); 200 break; 201 case MLX5_CMD_OP_MODIFY_RQ: 202 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 203 MLX5_GET(modify_rq_in, in, rqn)); 204 break; 205 case MLX5_CMD_OP_QUERY_RMP: 206 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP, 207 MLX5_GET(query_rmp_in, in, rmpn)); 208 break; 209 case MLX5_CMD_OP_MODIFY_RMP: 210 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP, 211 MLX5_GET(modify_rmp_in, in, rmpn)); 212 break; 213 case MLX5_CMD_OP_QUERY_RQT: 214 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT, 215 MLX5_GET(query_rqt_in, in, rqtn)); 216 break; 217 case MLX5_CMD_OP_MODIFY_RQT: 218 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT, 219 MLX5_GET(modify_rqt_in, in, rqtn)); 220 break; 221 case MLX5_CMD_OP_QUERY_TIR: 222 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR, 223 MLX5_GET(query_tir_in, in, tirn)); 224 break; 225 case MLX5_CMD_OP_MODIFY_TIR: 226 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR, 227 MLX5_GET(modify_tir_in, in, tirn)); 228 break; 229 case MLX5_CMD_OP_QUERY_TIS: 230 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS, 231 MLX5_GET(query_tis_in, in, tisn)); 232 break; 233 case MLX5_CMD_OP_MODIFY_TIS: 234 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS, 235 MLX5_GET(modify_tis_in, in, tisn)); 236 break; 237 case MLX5_CMD_OP_QUERY_FLOW_TABLE: 238 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE, 239 MLX5_GET(query_flow_table_in, in, 240 table_id)); 241 break; 242 case MLX5_CMD_OP_MODIFY_FLOW_TABLE: 243 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE, 244 MLX5_GET(modify_flow_table_in, in, 245 table_id)); 246 break; 247 case MLX5_CMD_OP_QUERY_FLOW_GROUP: 248 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP, 249 MLX5_GET(query_flow_group_in, in, 250 group_id)); 251 break; 252 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: 253 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY, 254 MLX5_GET(query_fte_in, in, 255 flow_index)); 256 break; 257 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 258 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY, 259 MLX5_GET(set_fte_in, in, flow_index)); 260 break; 261 case MLX5_CMD_OP_QUERY_Q_COUNTER: 262 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER, 263 MLX5_GET(query_q_counter_in, in, 264 counter_set_id)); 265 break; 266 case MLX5_CMD_OP_QUERY_FLOW_COUNTER: 267 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER, 268 MLX5_GET(query_flow_counter_in, in, 269 flow_counter_id)); 270 break; 271 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT: 272 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT, 273 MLX5_GET(general_obj_in_cmd_hdr, in, 274 obj_id)); 275 break; 276 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT: 277 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT, 278 MLX5_GET(query_scheduling_element_in, 279 in, scheduling_element_id)); 280 break; 281 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT: 282 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT, 283 MLX5_GET(modify_scheduling_element_in, 284 in, scheduling_element_id)); 285 break; 286 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 287 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT, 288 MLX5_GET(add_vxlan_udp_dport_in, in, 289 vxlan_udp_port)); 290 break; 291 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: 292 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY, 293 MLX5_GET(query_l2_table_entry_in, in, 294 table_index)); 295 break; 296 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 297 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY, 298 MLX5_GET(set_l2_table_entry_in, in, 299 table_index)); 300 break; 301 case MLX5_CMD_OP_QUERY_QP: 302 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 303 MLX5_GET(query_qp_in, in, qpn)); 304 break; 305 case MLX5_CMD_OP_RST2INIT_QP: 306 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 307 MLX5_GET(rst2init_qp_in, in, qpn)); 308 break; 309 case MLX5_CMD_OP_INIT2RTR_QP: 310 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 311 MLX5_GET(init2rtr_qp_in, in, qpn)); 312 break; 313 case MLX5_CMD_OP_RTR2RTS_QP: 314 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 315 MLX5_GET(rtr2rts_qp_in, in, qpn)); 316 break; 317 case MLX5_CMD_OP_RTS2RTS_QP: 318 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 319 MLX5_GET(rts2rts_qp_in, in, qpn)); 320 break; 321 case MLX5_CMD_OP_SQERR2RTS_QP: 322 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 323 MLX5_GET(sqerr2rts_qp_in, in, qpn)); 324 break; 325 case MLX5_CMD_OP_2ERR_QP: 326 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 327 MLX5_GET(qp_2err_in, in, qpn)); 328 break; 329 case MLX5_CMD_OP_2RST_QP: 330 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 331 MLX5_GET(qp_2rst_in, in, qpn)); 332 break; 333 case MLX5_CMD_OP_QUERY_DCT: 334 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT, 335 MLX5_GET(query_dct_in, in, dctn)); 336 break; 337 case MLX5_CMD_OP_QUERY_XRQ: 338 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY: 339 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS: 340 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ, 341 MLX5_GET(query_xrq_in, in, xrqn)); 342 break; 343 case MLX5_CMD_OP_QUERY_XRC_SRQ: 344 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ, 345 MLX5_GET(query_xrc_srq_in, in, 346 xrc_srqn)); 347 break; 348 case MLX5_CMD_OP_ARM_XRC_SRQ: 349 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ, 350 MLX5_GET(arm_xrc_srq_in, in, xrc_srqn)); 351 break; 352 case MLX5_CMD_OP_QUERY_SRQ: 353 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ, 354 MLX5_GET(query_srq_in, in, srqn)); 355 break; 356 case MLX5_CMD_OP_ARM_RQ: 357 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 358 MLX5_GET(arm_rq_in, in, srq_number)); 359 break; 360 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: 361 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT, 362 MLX5_GET(drain_dct_in, in, dctn)); 363 break; 364 case MLX5_CMD_OP_ARM_XRQ: 365 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY: 366 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ, 367 MLX5_GET(arm_xrq_in, in, xrqn)); 368 break; 369 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT: 370 obj_id = get_enc_obj_id 371 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT, 372 MLX5_GET(query_packet_reformat_context_in, 373 in, packet_reformat_id)); 374 break; 375 default: 376 obj_id = 0; 377 } 378 379 return obj_id; 380 } 381 382 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs, 383 struct ib_uobject *uobj, const void *in) 384 { 385 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 386 u64 obj_id = devx_get_obj_id(in); 387 388 if (!obj_id) 389 return false; 390 391 switch (uobj_get_object_id(uobj)) { 392 case UVERBS_OBJECT_CQ: 393 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ, 394 to_mcq(uobj->object)->mcq.cqn) == 395 obj_id; 396 397 case UVERBS_OBJECT_SRQ: 398 { 399 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq); 400 u16 opcode; 401 402 switch (srq->common.res) { 403 case MLX5_RES_XSRQ: 404 opcode = MLX5_CMD_OP_CREATE_XRC_SRQ; 405 break; 406 case MLX5_RES_XRQ: 407 opcode = MLX5_CMD_OP_CREATE_XRQ; 408 break; 409 default: 410 if (!dev->mdev->issi) 411 opcode = MLX5_CMD_OP_CREATE_SRQ; 412 else 413 opcode = MLX5_CMD_OP_CREATE_RMP; 414 } 415 416 return get_enc_obj_id(opcode, 417 to_msrq(uobj->object)->msrq.srqn) == 418 obj_id; 419 } 420 421 case UVERBS_OBJECT_QP: 422 { 423 struct mlx5_ib_qp *qp = to_mqp(uobj->object); 424 enum ib_qp_type qp_type = qp->ibqp.qp_type; 425 426 if (qp_type == IB_QPT_RAW_PACKET || 427 (qp->flags & MLX5_IB_QP_UNDERLAY)) { 428 struct mlx5_ib_raw_packet_qp *raw_packet_qp = 429 &qp->raw_packet_qp; 430 struct mlx5_ib_rq *rq = &raw_packet_qp->rq; 431 struct mlx5_ib_sq *sq = &raw_packet_qp->sq; 432 433 return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 434 rq->base.mqp.qpn) == obj_id || 435 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ, 436 sq->base.mqp.qpn) == obj_id || 437 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR, 438 rq->tirn) == obj_id || 439 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS, 440 sq->tisn) == obj_id); 441 } 442 443 if (qp_type == MLX5_IB_QPT_DCT) 444 return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT, 445 qp->dct.mdct.mqp.qpn) == obj_id; 446 447 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 448 qp->ibqp.qp_num) == obj_id; 449 } 450 451 case UVERBS_OBJECT_WQ: 452 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 453 to_mrwq(uobj->object)->core_qp.qpn) == 454 obj_id; 455 456 case UVERBS_OBJECT_RWQ_IND_TBL: 457 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT, 458 to_mrwq_ind_table(uobj->object)->rqtn) == 459 obj_id; 460 461 case MLX5_IB_OBJECT_DEVX_OBJ: 462 return ((struct devx_obj *)uobj->object)->obj_id == obj_id; 463 464 default: 465 return false; 466 } 467 } 468 469 static void devx_set_umem_valid(const void *in) 470 { 471 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 472 473 switch (opcode) { 474 case MLX5_CMD_OP_CREATE_MKEY: 475 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1); 476 break; 477 case MLX5_CMD_OP_CREATE_CQ: 478 { 479 void *cqc; 480 481 MLX5_SET(create_cq_in, in, cq_umem_valid, 1); 482 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 483 MLX5_SET(cqc, cqc, dbr_umem_valid, 1); 484 break; 485 } 486 case MLX5_CMD_OP_CREATE_QP: 487 { 488 void *qpc; 489 490 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 491 MLX5_SET(qpc, qpc, dbr_umem_valid, 1); 492 MLX5_SET(create_qp_in, in, wq_umem_valid, 1); 493 break; 494 } 495 496 case MLX5_CMD_OP_CREATE_RQ: 497 { 498 void *rqc, *wq; 499 500 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 501 wq = MLX5_ADDR_OF(rqc, rqc, wq); 502 MLX5_SET(wq, wq, dbr_umem_valid, 1); 503 MLX5_SET(wq, wq, wq_umem_valid, 1); 504 break; 505 } 506 507 case MLX5_CMD_OP_CREATE_SQ: 508 { 509 void *sqc, *wq; 510 511 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 512 wq = MLX5_ADDR_OF(sqc, sqc, wq); 513 MLX5_SET(wq, wq, dbr_umem_valid, 1); 514 MLX5_SET(wq, wq, wq_umem_valid, 1); 515 break; 516 } 517 518 case MLX5_CMD_OP_MODIFY_CQ: 519 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1); 520 break; 521 522 case MLX5_CMD_OP_CREATE_RMP: 523 { 524 void *rmpc, *wq; 525 526 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx); 527 wq = MLX5_ADDR_OF(rmpc, rmpc, wq); 528 MLX5_SET(wq, wq, dbr_umem_valid, 1); 529 MLX5_SET(wq, wq, wq_umem_valid, 1); 530 break; 531 } 532 533 case MLX5_CMD_OP_CREATE_XRQ: 534 { 535 void *xrqc, *wq; 536 537 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context); 538 wq = MLX5_ADDR_OF(xrqc, xrqc, wq); 539 MLX5_SET(wq, wq, dbr_umem_valid, 1); 540 MLX5_SET(wq, wq, wq_umem_valid, 1); 541 break; 542 } 543 544 case MLX5_CMD_OP_CREATE_XRC_SRQ: 545 { 546 void *xrc_srqc; 547 548 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1); 549 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in, 550 xrc_srq_context_entry); 551 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1); 552 break; 553 } 554 555 default: 556 return; 557 } 558 } 559 560 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode) 561 { 562 *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 563 564 switch (*opcode) { 565 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 566 case MLX5_CMD_OP_CREATE_MKEY: 567 case MLX5_CMD_OP_CREATE_CQ: 568 case MLX5_CMD_OP_ALLOC_PD: 569 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: 570 case MLX5_CMD_OP_CREATE_RMP: 571 case MLX5_CMD_OP_CREATE_SQ: 572 case MLX5_CMD_OP_CREATE_RQ: 573 case MLX5_CMD_OP_CREATE_RQT: 574 case MLX5_CMD_OP_CREATE_TIR: 575 case MLX5_CMD_OP_CREATE_TIS: 576 case MLX5_CMD_OP_ALLOC_Q_COUNTER: 577 case MLX5_CMD_OP_CREATE_FLOW_TABLE: 578 case MLX5_CMD_OP_CREATE_FLOW_GROUP: 579 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: 580 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT: 581 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT: 582 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT: 583 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 584 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 585 case MLX5_CMD_OP_CREATE_QP: 586 case MLX5_CMD_OP_CREATE_SRQ: 587 case MLX5_CMD_OP_CREATE_XRC_SRQ: 588 case MLX5_CMD_OP_CREATE_DCT: 589 case MLX5_CMD_OP_CREATE_XRQ: 590 case MLX5_CMD_OP_ATTACH_TO_MCG: 591 case MLX5_CMD_OP_ALLOC_XRCD: 592 return true; 593 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 594 { 595 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod); 596 if (op_mod == 0) 597 return true; 598 return false; 599 } 600 default: 601 return false; 602 } 603 } 604 605 static bool devx_is_obj_modify_cmd(const void *in) 606 { 607 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 608 609 switch (opcode) { 610 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 611 case MLX5_CMD_OP_MODIFY_CQ: 612 case MLX5_CMD_OP_MODIFY_RMP: 613 case MLX5_CMD_OP_MODIFY_SQ: 614 case MLX5_CMD_OP_MODIFY_RQ: 615 case MLX5_CMD_OP_MODIFY_RQT: 616 case MLX5_CMD_OP_MODIFY_TIR: 617 case MLX5_CMD_OP_MODIFY_TIS: 618 case MLX5_CMD_OP_MODIFY_FLOW_TABLE: 619 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT: 620 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 621 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 622 case MLX5_CMD_OP_RST2INIT_QP: 623 case MLX5_CMD_OP_INIT2RTR_QP: 624 case MLX5_CMD_OP_RTR2RTS_QP: 625 case MLX5_CMD_OP_RTS2RTS_QP: 626 case MLX5_CMD_OP_SQERR2RTS_QP: 627 case MLX5_CMD_OP_2ERR_QP: 628 case MLX5_CMD_OP_2RST_QP: 629 case MLX5_CMD_OP_ARM_XRC_SRQ: 630 case MLX5_CMD_OP_ARM_RQ: 631 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: 632 case MLX5_CMD_OP_ARM_XRQ: 633 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY: 634 return true; 635 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 636 { 637 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod); 638 639 if (op_mod == 1) 640 return true; 641 return false; 642 } 643 default: 644 return false; 645 } 646 } 647 648 static bool devx_is_obj_query_cmd(const void *in) 649 { 650 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 651 652 switch (opcode) { 653 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 654 case MLX5_CMD_OP_QUERY_MKEY: 655 case MLX5_CMD_OP_QUERY_CQ: 656 case MLX5_CMD_OP_QUERY_RMP: 657 case MLX5_CMD_OP_QUERY_SQ: 658 case MLX5_CMD_OP_QUERY_RQ: 659 case MLX5_CMD_OP_QUERY_RQT: 660 case MLX5_CMD_OP_QUERY_TIR: 661 case MLX5_CMD_OP_QUERY_TIS: 662 case MLX5_CMD_OP_QUERY_Q_COUNTER: 663 case MLX5_CMD_OP_QUERY_FLOW_TABLE: 664 case MLX5_CMD_OP_QUERY_FLOW_GROUP: 665 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: 666 case MLX5_CMD_OP_QUERY_FLOW_COUNTER: 667 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT: 668 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT: 669 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: 670 case MLX5_CMD_OP_QUERY_QP: 671 case MLX5_CMD_OP_QUERY_SRQ: 672 case MLX5_CMD_OP_QUERY_XRC_SRQ: 673 case MLX5_CMD_OP_QUERY_DCT: 674 case MLX5_CMD_OP_QUERY_XRQ: 675 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY: 676 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS: 677 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT: 678 return true; 679 default: 680 return false; 681 } 682 } 683 684 static bool devx_is_whitelist_cmd(void *in) 685 { 686 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 687 688 switch (opcode) { 689 case MLX5_CMD_OP_QUERY_HCA_CAP: 690 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: 691 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT: 692 return true; 693 default: 694 return false; 695 } 696 } 697 698 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in) 699 { 700 if (devx_is_whitelist_cmd(cmd_in)) { 701 struct mlx5_ib_dev *dev; 702 703 if (c->devx_uid) 704 return c->devx_uid; 705 706 dev = to_mdev(c->ibucontext.device); 707 if (dev->devx_whitelist_uid) 708 return dev->devx_whitelist_uid; 709 710 return -EOPNOTSUPP; 711 } 712 713 if (!c->devx_uid) 714 return -EINVAL; 715 716 return c->devx_uid; 717 } 718 static bool devx_is_general_cmd(void *in) 719 { 720 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 721 722 if (opcode >= MLX5_CMD_OP_GENERAL_START && 723 opcode < MLX5_CMD_OP_GENERAL_END) 724 return true; 725 726 switch (opcode) { 727 case MLX5_CMD_OP_QUERY_HCA_CAP: 728 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: 729 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT: 730 case MLX5_CMD_OP_QUERY_VPORT_STATE: 731 case MLX5_CMD_OP_QUERY_ADAPTER: 732 case MLX5_CMD_OP_QUERY_ISSI: 733 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT: 734 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS: 735 case MLX5_CMD_OP_QUERY_VNIC_ENV: 736 case MLX5_CMD_OP_QUERY_VPORT_COUNTER: 737 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG: 738 case MLX5_CMD_OP_NOP: 739 case MLX5_CMD_OP_QUERY_CONG_STATUS: 740 case MLX5_CMD_OP_QUERY_CONG_PARAMS: 741 case MLX5_CMD_OP_QUERY_CONG_STATISTICS: 742 return true; 743 default: 744 return false; 745 } 746 } 747 748 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)( 749 struct uverbs_attr_bundle *attrs) 750 { 751 struct mlx5_ib_ucontext *c; 752 struct mlx5_ib_dev *dev; 753 int user_vector; 754 int dev_eqn; 755 unsigned int irqn; 756 int err; 757 758 if (uverbs_copy_from(&user_vector, attrs, 759 MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC)) 760 return -EFAULT; 761 762 c = devx_ufile2uctx(attrs); 763 if (IS_ERR(c)) 764 return PTR_ERR(c); 765 dev = to_mdev(c->ibucontext.device); 766 767 err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn); 768 if (err < 0) 769 return err; 770 771 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN, 772 &dev_eqn, sizeof(dev_eqn))) 773 return -EFAULT; 774 775 return 0; 776 } 777 778 /* 779 *Security note: 780 * The hardware protection mechanism works like this: Each device object that 781 * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in 782 * the device specification manual) upon its creation. Then upon doorbell, 783 * hardware fetches the object context for which the doorbell was rang, and 784 * validates that the UAR through which the DB was rang matches the UAR ID 785 * of the object. 786 * If no match the doorbell is silently ignored by the hardware. Of course, 787 * the user cannot ring a doorbell on a UAR that was not mapped to it. 788 * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command 789 * mailboxes (except tagging them with UID), we expose to the user its UAR 790 * ID, so it can embed it in these objects in the expected specification 791 * format. So the only thing the user can do is hurt itself by creating a 792 * QP/SQ/CQ with a UAR ID other than his, and then in this case other users 793 * may ring a doorbell on its objects. 794 * The consequence of that will be that another user can schedule a QP/SQ 795 * of the buggy user for execution (just insert it to the hardware schedule 796 * queue or arm its CQ for event generation), no further harm is expected. 797 */ 798 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)( 799 struct uverbs_attr_bundle *attrs) 800 { 801 struct mlx5_ib_ucontext *c; 802 struct mlx5_ib_dev *dev; 803 u32 user_idx; 804 s32 dev_idx; 805 806 c = devx_ufile2uctx(attrs); 807 if (IS_ERR(c)) 808 return PTR_ERR(c); 809 dev = to_mdev(c->ibucontext.device); 810 811 if (uverbs_copy_from(&user_idx, attrs, 812 MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX)) 813 return -EFAULT; 814 815 dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true); 816 if (dev_idx < 0) 817 return dev_idx; 818 819 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX, 820 &dev_idx, sizeof(dev_idx))) 821 return -EFAULT; 822 823 return 0; 824 } 825 826 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)( 827 struct uverbs_attr_bundle *attrs) 828 { 829 struct mlx5_ib_ucontext *c; 830 struct mlx5_ib_dev *dev; 831 void *cmd_in = uverbs_attr_get_alloced_ptr( 832 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN); 833 int cmd_out_len = uverbs_attr_get_len(attrs, 834 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT); 835 void *cmd_out; 836 int err; 837 int uid; 838 839 c = devx_ufile2uctx(attrs); 840 if (IS_ERR(c)) 841 return PTR_ERR(c); 842 dev = to_mdev(c->ibucontext.device); 843 844 uid = devx_get_uid(c, cmd_in); 845 if (uid < 0) 846 return uid; 847 848 /* Only white list of some general HCA commands are allowed for this method. */ 849 if (!devx_is_general_cmd(cmd_in)) 850 return -EINVAL; 851 852 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 853 if (IS_ERR(cmd_out)) 854 return PTR_ERR(cmd_out); 855 856 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 857 err = mlx5_cmd_exec(dev->mdev, cmd_in, 858 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN), 859 cmd_out, cmd_out_len); 860 if (err) 861 return err; 862 863 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out, 864 cmd_out_len); 865 } 866 867 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din, 868 u32 *dinlen, 869 u32 *obj_id) 870 { 871 u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type); 872 u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid); 873 874 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 875 *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr); 876 877 MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id); 878 MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid); 879 880 switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) { 881 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 882 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 883 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type); 884 break; 885 886 case MLX5_CMD_OP_CREATE_UMEM: 887 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 888 MLX5_CMD_OP_DESTROY_UMEM); 889 break; 890 case MLX5_CMD_OP_CREATE_MKEY: 891 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY); 892 break; 893 case MLX5_CMD_OP_CREATE_CQ: 894 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ); 895 break; 896 case MLX5_CMD_OP_ALLOC_PD: 897 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD); 898 break; 899 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: 900 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 901 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN); 902 break; 903 case MLX5_CMD_OP_CREATE_RMP: 904 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP); 905 break; 906 case MLX5_CMD_OP_CREATE_SQ: 907 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ); 908 break; 909 case MLX5_CMD_OP_CREATE_RQ: 910 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ); 911 break; 912 case MLX5_CMD_OP_CREATE_RQT: 913 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT); 914 break; 915 case MLX5_CMD_OP_CREATE_TIR: 916 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR); 917 break; 918 case MLX5_CMD_OP_CREATE_TIS: 919 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS); 920 break; 921 case MLX5_CMD_OP_ALLOC_Q_COUNTER: 922 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 923 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 924 break; 925 case MLX5_CMD_OP_CREATE_FLOW_TABLE: 926 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in); 927 *obj_id = MLX5_GET(create_flow_table_out, out, table_id); 928 MLX5_SET(destroy_flow_table_in, din, other_vport, 929 MLX5_GET(create_flow_table_in, in, other_vport)); 930 MLX5_SET(destroy_flow_table_in, din, vport_number, 931 MLX5_GET(create_flow_table_in, in, vport_number)); 932 MLX5_SET(destroy_flow_table_in, din, table_type, 933 MLX5_GET(create_flow_table_in, in, table_type)); 934 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id); 935 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 936 MLX5_CMD_OP_DESTROY_FLOW_TABLE); 937 break; 938 case MLX5_CMD_OP_CREATE_FLOW_GROUP: 939 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in); 940 *obj_id = MLX5_GET(create_flow_group_out, out, group_id); 941 MLX5_SET(destroy_flow_group_in, din, other_vport, 942 MLX5_GET(create_flow_group_in, in, other_vport)); 943 MLX5_SET(destroy_flow_group_in, din, vport_number, 944 MLX5_GET(create_flow_group_in, in, vport_number)); 945 MLX5_SET(destroy_flow_group_in, din, table_type, 946 MLX5_GET(create_flow_group_in, in, table_type)); 947 MLX5_SET(destroy_flow_group_in, din, table_id, 948 MLX5_GET(create_flow_group_in, in, table_id)); 949 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id); 950 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 951 MLX5_CMD_OP_DESTROY_FLOW_GROUP); 952 break; 953 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 954 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in); 955 *obj_id = MLX5_GET(set_fte_in, in, flow_index); 956 MLX5_SET(delete_fte_in, din, other_vport, 957 MLX5_GET(set_fte_in, in, other_vport)); 958 MLX5_SET(delete_fte_in, din, vport_number, 959 MLX5_GET(set_fte_in, in, vport_number)); 960 MLX5_SET(delete_fte_in, din, table_type, 961 MLX5_GET(set_fte_in, in, table_type)); 962 MLX5_SET(delete_fte_in, din, table_id, 963 MLX5_GET(set_fte_in, in, table_id)); 964 MLX5_SET(delete_fte_in, din, flow_index, *obj_id); 965 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 966 MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY); 967 break; 968 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: 969 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 970 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER); 971 break; 972 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT: 973 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 974 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT); 975 break; 976 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT: 977 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 978 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT); 979 break; 980 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT: 981 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in); 982 *obj_id = MLX5_GET(create_scheduling_element_out, out, 983 scheduling_element_id); 984 MLX5_SET(destroy_scheduling_element_in, din, 985 scheduling_hierarchy, 986 MLX5_GET(create_scheduling_element_in, in, 987 scheduling_hierarchy)); 988 MLX5_SET(destroy_scheduling_element_in, din, 989 scheduling_element_id, *obj_id); 990 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 991 MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT); 992 break; 993 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 994 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in); 995 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port); 996 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id); 997 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 998 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT); 999 break; 1000 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 1001 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in); 1002 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index); 1003 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id); 1004 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 1005 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY); 1006 break; 1007 case MLX5_CMD_OP_CREATE_QP: 1008 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP); 1009 break; 1010 case MLX5_CMD_OP_CREATE_SRQ: 1011 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ); 1012 break; 1013 case MLX5_CMD_OP_CREATE_XRC_SRQ: 1014 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 1015 MLX5_CMD_OP_DESTROY_XRC_SRQ); 1016 break; 1017 case MLX5_CMD_OP_CREATE_DCT: 1018 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT); 1019 break; 1020 case MLX5_CMD_OP_CREATE_XRQ: 1021 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ); 1022 break; 1023 case MLX5_CMD_OP_ATTACH_TO_MCG: 1024 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in); 1025 MLX5_SET(detach_from_mcg_in, din, qpn, 1026 MLX5_GET(attach_to_mcg_in, in, qpn)); 1027 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid), 1028 MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid), 1029 MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid)); 1030 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG); 1031 break; 1032 case MLX5_CMD_OP_ALLOC_XRCD: 1033 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD); 1034 break; 1035 default: 1036 /* The entry must match to one of the devx_is_obj_create_cmd */ 1037 WARN_ON(true); 1038 break; 1039 } 1040 } 1041 1042 static int devx_handle_mkey_indirect(struct devx_obj *obj, 1043 struct mlx5_ib_dev *dev, 1044 void *in, void *out) 1045 { 1046 struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr; 1047 struct mlx5_core_mkey *mkey; 1048 void *mkc; 1049 u8 key; 1050 1051 mkey = &devx_mr->mmkey; 1052 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1053 key = MLX5_GET(mkc, mkc, mkey_7_0); 1054 mkey->key = mlx5_idx_to_mkey( 1055 MLX5_GET(create_mkey_out, out, mkey_index)) | key; 1056 mkey->type = MLX5_MKEY_INDIRECT_DEVX; 1057 mkey->iova = MLX5_GET64(mkc, mkc, start_addr); 1058 mkey->size = MLX5_GET64(mkc, mkc, len); 1059 mkey->pd = MLX5_GET(mkc, mkc, pd); 1060 devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size); 1061 1062 return xa_err(xa_store(&dev->mdev->priv.mkey_table, 1063 mlx5_base_mkey(mkey->key), mkey, GFP_KERNEL)); 1064 } 1065 1066 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev, 1067 struct devx_obj *obj, 1068 void *in, int in_len) 1069 { 1070 int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) + 1071 MLX5_FLD_SZ_BYTES(create_mkey_in, 1072 memory_key_mkey_entry); 1073 void *mkc; 1074 u8 access_mode; 1075 1076 if (in_len < min_len) 1077 return -EINVAL; 1078 1079 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1080 1081 access_mode = MLX5_GET(mkc, mkc, access_mode_1_0); 1082 access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2; 1083 1084 if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS || 1085 access_mode == MLX5_MKC_ACCESS_MODE_KSM) { 1086 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 1087 obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY; 1088 return 0; 1089 } 1090 1091 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1); 1092 return 0; 1093 } 1094 1095 static void devx_free_indirect_mkey(struct rcu_head *rcu) 1096 { 1097 kfree(container_of(rcu, struct devx_obj, devx_mr.rcu)); 1098 } 1099 1100 /* This function to delete from the radix tree needs to be called before 1101 * destroying the underlying mkey. Otherwise a race might occur in case that 1102 * other thread will get the same mkey before this one will be deleted, 1103 * in that case it will fail via inserting to the tree its own data. 1104 * 1105 * Note: 1106 * An error in the destroy is not expected unless there is some other indirect 1107 * mkey which points to this one. In a kernel cleanup flow it will be just 1108 * destroyed in the iterative destruction call. In a user flow, in case 1109 * the application didn't close in the expected order it's its own problem, 1110 * the mkey won't be part of the tree, in both cases the kernel is safe. 1111 */ 1112 static void devx_cleanup_mkey(struct devx_obj *obj) 1113 { 1114 xa_erase(&obj->mdev->priv.mkey_table, 1115 mlx5_base_mkey(obj->devx_mr.mmkey.key)); 1116 } 1117 1118 static int devx_obj_cleanup(struct ib_uobject *uobject, 1119 enum rdma_remove_reason why, 1120 struct uverbs_attr_bundle *attrs) 1121 { 1122 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 1123 struct devx_obj *obj = uobject->object; 1124 int ret; 1125 1126 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) 1127 devx_cleanup_mkey(obj); 1128 1129 if (obj->flags & DEVX_OBJ_FLAGS_DCT) 1130 ret = mlx5_core_destroy_dct(obj->mdev, &obj->core_dct); 1131 else 1132 ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, 1133 sizeof(out)); 1134 if (ib_is_destroy_retryable(ret, why, uobject)) 1135 return ret; 1136 1137 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) { 1138 struct mlx5_ib_dev *dev = 1139 mlx5_udata_to_mdev(&attrs->driver_udata); 1140 1141 call_srcu(&dev->mr_srcu, &obj->devx_mr.rcu, 1142 devx_free_indirect_mkey); 1143 return ret; 1144 } 1145 1146 kfree(obj); 1147 return ret; 1148 } 1149 1150 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( 1151 struct uverbs_attr_bundle *attrs) 1152 { 1153 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN); 1154 int cmd_out_len = uverbs_attr_get_len(attrs, 1155 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT); 1156 int cmd_in_len = uverbs_attr_get_len(attrs, 1157 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN); 1158 void *cmd_out; 1159 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1160 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE); 1161 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1162 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1163 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); 1164 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 1165 struct devx_obj *obj; 1166 u16 obj_type = 0; 1167 int err; 1168 int uid; 1169 u32 obj_id; 1170 u16 opcode; 1171 1172 uid = devx_get_uid(c, cmd_in); 1173 if (uid < 0) 1174 return uid; 1175 1176 if (!devx_is_obj_create_cmd(cmd_in, &opcode)) 1177 return -EINVAL; 1178 1179 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 1180 if (IS_ERR(cmd_out)) 1181 return PTR_ERR(cmd_out); 1182 1183 obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL); 1184 if (!obj) 1185 return -ENOMEM; 1186 1187 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1188 if (opcode == MLX5_CMD_OP_CREATE_MKEY) { 1189 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len); 1190 if (err) 1191 goto obj_free; 1192 } else { 1193 devx_set_umem_valid(cmd_in); 1194 } 1195 1196 if (opcode == MLX5_CMD_OP_CREATE_DCT) { 1197 obj->flags |= DEVX_OBJ_FLAGS_DCT; 1198 err = mlx5_core_create_dct(dev->mdev, &obj->core_dct, 1199 cmd_in, cmd_in_len, 1200 cmd_out, cmd_out_len); 1201 } else { 1202 err = mlx5_cmd_exec(dev->mdev, cmd_in, 1203 cmd_in_len, 1204 cmd_out, cmd_out_len); 1205 } 1206 1207 if (err) 1208 goto obj_free; 1209 1210 uobj->object = obj; 1211 obj->mdev = dev->mdev; 1212 devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen, 1213 &obj_id); 1214 WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32)); 1215 1216 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) { 1217 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out); 1218 if (err) 1219 goto obj_destroy; 1220 } 1221 1222 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len); 1223 if (err) 1224 goto err_copy; 1225 1226 if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT) 1227 obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type); 1228 1229 obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id); 1230 1231 return 0; 1232 1233 err_copy: 1234 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) 1235 devx_cleanup_mkey(obj); 1236 obj_destroy: 1237 if (obj->flags & DEVX_OBJ_FLAGS_DCT) 1238 mlx5_core_destroy_dct(obj->mdev, &obj->core_dct); 1239 else 1240 mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, 1241 sizeof(out)); 1242 obj_free: 1243 kfree(obj); 1244 return err; 1245 } 1246 1247 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)( 1248 struct uverbs_attr_bundle *attrs) 1249 { 1250 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN); 1251 int cmd_out_len = uverbs_attr_get_len(attrs, 1252 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT); 1253 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 1254 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE); 1255 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1256 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1257 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); 1258 void *cmd_out; 1259 int err; 1260 int uid; 1261 1262 uid = devx_get_uid(c, cmd_in); 1263 if (uid < 0) 1264 return uid; 1265 1266 if (!devx_is_obj_modify_cmd(cmd_in)) 1267 return -EINVAL; 1268 1269 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in)) 1270 return -EINVAL; 1271 1272 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 1273 if (IS_ERR(cmd_out)) 1274 return PTR_ERR(cmd_out); 1275 1276 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1277 devx_set_umem_valid(cmd_in); 1278 1279 err = mlx5_cmd_exec(mdev->mdev, cmd_in, 1280 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN), 1281 cmd_out, cmd_out_len); 1282 if (err) 1283 return err; 1284 1285 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT, 1286 cmd_out, cmd_out_len); 1287 } 1288 1289 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)( 1290 struct uverbs_attr_bundle *attrs) 1291 { 1292 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN); 1293 int cmd_out_len = uverbs_attr_get_len(attrs, 1294 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT); 1295 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 1296 MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE); 1297 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1298 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1299 void *cmd_out; 1300 int err; 1301 int uid; 1302 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); 1303 1304 uid = devx_get_uid(c, cmd_in); 1305 if (uid < 0) 1306 return uid; 1307 1308 if (!devx_is_obj_query_cmd(cmd_in)) 1309 return -EINVAL; 1310 1311 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in)) 1312 return -EINVAL; 1313 1314 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 1315 if (IS_ERR(cmd_out)) 1316 return PTR_ERR(cmd_out); 1317 1318 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1319 err = mlx5_cmd_exec(mdev->mdev, cmd_in, 1320 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN), 1321 cmd_out, cmd_out_len); 1322 if (err) 1323 return err; 1324 1325 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT, 1326 cmd_out, cmd_out_len); 1327 } 1328 1329 struct devx_async_event_queue { 1330 spinlock_t lock; 1331 wait_queue_head_t poll_wait; 1332 struct list_head event_list; 1333 atomic_t bytes_in_use; 1334 u8 is_destroyed:1; 1335 }; 1336 1337 struct devx_async_cmd_event_file { 1338 struct ib_uobject uobj; 1339 struct devx_async_event_queue ev_queue; 1340 struct mlx5_async_ctx async_ctx; 1341 }; 1342 1343 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue) 1344 { 1345 spin_lock_init(&ev_queue->lock); 1346 INIT_LIST_HEAD(&ev_queue->event_list); 1347 init_waitqueue_head(&ev_queue->poll_wait); 1348 atomic_set(&ev_queue->bytes_in_use, 0); 1349 ev_queue->is_destroyed = 0; 1350 } 1351 1352 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)( 1353 struct uverbs_attr_bundle *attrs) 1354 { 1355 struct devx_async_cmd_event_file *ev_file; 1356 1357 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1358 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE); 1359 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 1360 1361 ev_file = container_of(uobj, struct devx_async_cmd_event_file, 1362 uobj); 1363 devx_init_event_queue(&ev_file->ev_queue); 1364 mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx); 1365 return 0; 1366 } 1367 1368 static void devx_query_callback(int status, struct mlx5_async_work *context) 1369 { 1370 struct devx_async_data *async_data = 1371 container_of(context, struct devx_async_data, cb_work); 1372 struct ib_uobject *fd_uobj = async_data->fd_uobj; 1373 struct devx_async_cmd_event_file *ev_file; 1374 struct devx_async_event_queue *ev_queue; 1375 unsigned long flags; 1376 1377 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file, 1378 uobj); 1379 ev_queue = &ev_file->ev_queue; 1380 1381 spin_lock_irqsave(&ev_queue->lock, flags); 1382 list_add_tail(&async_data->list, &ev_queue->event_list); 1383 spin_unlock_irqrestore(&ev_queue->lock, flags); 1384 1385 wake_up_interruptible(&ev_queue->poll_wait); 1386 fput(fd_uobj->object); 1387 } 1388 1389 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */ 1390 1391 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)( 1392 struct uverbs_attr_bundle *attrs) 1393 { 1394 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, 1395 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN); 1396 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1397 attrs, 1398 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE); 1399 u16 cmd_out_len; 1400 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1401 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1402 struct ib_uobject *fd_uobj; 1403 int err; 1404 int uid; 1405 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); 1406 struct devx_async_cmd_event_file *ev_file; 1407 struct devx_async_data *async_data; 1408 1409 uid = devx_get_uid(c, cmd_in); 1410 if (uid < 0) 1411 return uid; 1412 1413 if (!devx_is_obj_query_cmd(cmd_in)) 1414 return -EINVAL; 1415 1416 err = uverbs_get_const(&cmd_out_len, attrs, 1417 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN); 1418 if (err) 1419 return err; 1420 1421 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in)) 1422 return -EINVAL; 1423 1424 fd_uobj = uverbs_attr_get_uobject(attrs, 1425 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD); 1426 if (IS_ERR(fd_uobj)) 1427 return PTR_ERR(fd_uobj); 1428 1429 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file, 1430 uobj); 1431 1432 if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) > 1433 MAX_ASYNC_BYTES_IN_USE) { 1434 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use); 1435 return -EAGAIN; 1436 } 1437 1438 async_data = kvzalloc(struct_size(async_data, hdr.out_data, 1439 cmd_out_len), GFP_KERNEL); 1440 if (!async_data) { 1441 err = -ENOMEM; 1442 goto sub_bytes; 1443 } 1444 1445 err = uverbs_copy_from(&async_data->hdr.wr_id, attrs, 1446 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID); 1447 if (err) 1448 goto free_async; 1449 1450 async_data->cmd_out_len = cmd_out_len; 1451 async_data->mdev = mdev; 1452 async_data->fd_uobj = fd_uobj; 1453 1454 get_file(fd_uobj->object); 1455 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1456 err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in, 1457 uverbs_attr_get_len(attrs, 1458 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN), 1459 async_data->hdr.out_data, 1460 async_data->cmd_out_len, 1461 devx_query_callback, &async_data->cb_work); 1462 1463 if (err) 1464 goto cb_err; 1465 1466 return 0; 1467 1468 cb_err: 1469 fput(fd_uobj->object); 1470 free_async: 1471 kvfree(async_data); 1472 sub_bytes: 1473 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use); 1474 return err; 1475 } 1476 1477 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, 1478 struct uverbs_attr_bundle *attrs, 1479 struct devx_umem *obj) 1480 { 1481 u64 addr; 1482 size_t size; 1483 u32 access; 1484 int npages; 1485 int err; 1486 u32 page_mask; 1487 1488 if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) || 1489 uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN)) 1490 return -EFAULT; 1491 1492 err = uverbs_get_flags32(&access, attrs, 1493 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, 1494 IB_ACCESS_LOCAL_WRITE | 1495 IB_ACCESS_REMOTE_WRITE | 1496 IB_ACCESS_REMOTE_READ); 1497 if (err) 1498 return err; 1499 1500 err = ib_check_mr_access(access); 1501 if (err) 1502 return err; 1503 1504 obj->umem = ib_umem_get(&attrs->driver_udata, addr, size, access, 0); 1505 if (IS_ERR(obj->umem)) 1506 return PTR_ERR(obj->umem); 1507 1508 mlx5_ib_cont_pages(obj->umem, obj->umem->address, 1509 MLX5_MKEY_PAGE_SHIFT_MASK, &npages, 1510 &obj->page_shift, &obj->ncont, NULL); 1511 1512 if (!npages) { 1513 ib_umem_release(obj->umem); 1514 return -EINVAL; 1515 } 1516 1517 page_mask = (1 << obj->page_shift) - 1; 1518 obj->page_offset = obj->umem->address & page_mask; 1519 1520 return 0; 1521 } 1522 1523 static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs, 1524 struct devx_umem *obj, 1525 struct devx_umem_reg_cmd *cmd) 1526 { 1527 cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) + 1528 (MLX5_ST_SZ_BYTES(mtt) * obj->ncont); 1529 cmd->in = uverbs_zalloc(attrs, cmd->inlen); 1530 return PTR_ERR_OR_ZERO(cmd->in); 1531 } 1532 1533 static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev, 1534 struct devx_umem *obj, 1535 struct devx_umem_reg_cmd *cmd) 1536 { 1537 void *umem; 1538 __be64 *mtt; 1539 1540 umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem); 1541 mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt); 1542 1543 MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM); 1544 MLX5_SET64(umem, umem, num_of_mtt, obj->ncont); 1545 MLX5_SET(umem, umem, log_page_size, obj->page_shift - 1546 MLX5_ADAPTER_PAGE_SHIFT); 1547 MLX5_SET(umem, umem, page_offset, obj->page_offset); 1548 mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt, 1549 (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) | 1550 MLX5_IB_MTT_READ); 1551 } 1552 1553 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)( 1554 struct uverbs_attr_bundle *attrs) 1555 { 1556 struct devx_umem_reg_cmd cmd; 1557 struct devx_umem *obj; 1558 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1559 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE); 1560 u32 obj_id; 1561 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1562 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1563 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); 1564 int err; 1565 1566 if (!c->devx_uid) 1567 return -EINVAL; 1568 1569 obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL); 1570 if (!obj) 1571 return -ENOMEM; 1572 1573 err = devx_umem_get(dev, &c->ibucontext, attrs, obj); 1574 if (err) 1575 goto err_obj_free; 1576 1577 err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd); 1578 if (err) 1579 goto err_umem_release; 1580 1581 devx_umem_reg_cmd_build(dev, obj, &cmd); 1582 1583 MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid); 1584 err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out, 1585 sizeof(cmd.out)); 1586 if (err) 1587 goto err_umem_release; 1588 1589 obj->mdev = dev->mdev; 1590 uobj->object = obj; 1591 devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id); 1592 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id)); 1593 if (err) 1594 goto err_umem_destroy; 1595 1596 return 0; 1597 1598 err_umem_destroy: 1599 mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out)); 1600 err_umem_release: 1601 ib_umem_release(obj->umem); 1602 err_obj_free: 1603 kfree(obj); 1604 return err; 1605 } 1606 1607 static int devx_umem_cleanup(struct ib_uobject *uobject, 1608 enum rdma_remove_reason why, 1609 struct uverbs_attr_bundle *attrs) 1610 { 1611 struct devx_umem *obj = uobject->object; 1612 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 1613 int err; 1614 1615 err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out)); 1616 if (ib_is_destroy_retryable(err, why, uobject)) 1617 return err; 1618 1619 ib_umem_release(obj->umem); 1620 kfree(obj); 1621 return 0; 1622 } 1623 1624 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf, 1625 size_t count, loff_t *pos) 1626 { 1627 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data; 1628 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; 1629 struct devx_async_data *event; 1630 int ret = 0; 1631 size_t eventsz; 1632 1633 spin_lock_irq(&ev_queue->lock); 1634 1635 while (list_empty(&ev_queue->event_list)) { 1636 spin_unlock_irq(&ev_queue->lock); 1637 1638 if (filp->f_flags & O_NONBLOCK) 1639 return -EAGAIN; 1640 1641 if (wait_event_interruptible( 1642 ev_queue->poll_wait, 1643 (!list_empty(&ev_queue->event_list) || 1644 ev_queue->is_destroyed))) { 1645 return -ERESTARTSYS; 1646 } 1647 1648 if (list_empty(&ev_queue->event_list) && 1649 ev_queue->is_destroyed) 1650 return -EIO; 1651 1652 spin_lock_irq(&ev_queue->lock); 1653 } 1654 1655 event = list_entry(ev_queue->event_list.next, 1656 struct devx_async_data, list); 1657 eventsz = event->cmd_out_len + 1658 sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr); 1659 1660 if (eventsz > count) { 1661 spin_unlock_irq(&ev_queue->lock); 1662 return -ENOSPC; 1663 } 1664 1665 list_del(ev_queue->event_list.next); 1666 spin_unlock_irq(&ev_queue->lock); 1667 1668 if (copy_to_user(buf, &event->hdr, eventsz)) 1669 ret = -EFAULT; 1670 else 1671 ret = eventsz; 1672 1673 atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use); 1674 kvfree(event); 1675 return ret; 1676 } 1677 1678 static int devx_async_cmd_event_close(struct inode *inode, struct file *filp) 1679 { 1680 struct ib_uobject *uobj = filp->private_data; 1681 struct devx_async_cmd_event_file *comp_ev_file = container_of( 1682 uobj, struct devx_async_cmd_event_file, uobj); 1683 struct devx_async_data *entry, *tmp; 1684 1685 spin_lock_irq(&comp_ev_file->ev_queue.lock); 1686 list_for_each_entry_safe(entry, tmp, 1687 &comp_ev_file->ev_queue.event_list, list) 1688 kvfree(entry); 1689 spin_unlock_irq(&comp_ev_file->ev_queue.lock); 1690 1691 uverbs_close_fd(filp); 1692 return 0; 1693 } 1694 1695 static __poll_t devx_async_cmd_event_poll(struct file *filp, 1696 struct poll_table_struct *wait) 1697 { 1698 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data; 1699 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; 1700 __poll_t pollflags = 0; 1701 1702 poll_wait(filp, &ev_queue->poll_wait, wait); 1703 1704 spin_lock_irq(&ev_queue->lock); 1705 if (ev_queue->is_destroyed) 1706 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 1707 else if (!list_empty(&ev_queue->event_list)) 1708 pollflags = EPOLLIN | EPOLLRDNORM; 1709 spin_unlock_irq(&ev_queue->lock); 1710 1711 return pollflags; 1712 } 1713 1714 static const struct file_operations devx_async_cmd_event_fops = { 1715 .owner = THIS_MODULE, 1716 .read = devx_async_cmd_event_read, 1717 .poll = devx_async_cmd_event_poll, 1718 .release = devx_async_cmd_event_close, 1719 .llseek = no_llseek, 1720 }; 1721 1722 static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj, 1723 enum rdma_remove_reason why) 1724 { 1725 struct devx_async_cmd_event_file *comp_ev_file = 1726 container_of(uobj, struct devx_async_cmd_event_file, 1727 uobj); 1728 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; 1729 1730 spin_lock_irq(&ev_queue->lock); 1731 ev_queue->is_destroyed = 1; 1732 spin_unlock_irq(&ev_queue->lock); 1733 1734 if (why == RDMA_REMOVE_DRIVER_REMOVE) 1735 wake_up_interruptible(&ev_queue->poll_wait); 1736 1737 mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx); 1738 return 0; 1739 }; 1740 1741 DECLARE_UVERBS_NAMED_METHOD( 1742 MLX5_IB_METHOD_DEVX_UMEM_REG, 1743 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE, 1744 MLX5_IB_OBJECT_DEVX_UMEM, 1745 UVERBS_ACCESS_NEW, 1746 UA_MANDATORY), 1747 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR, 1748 UVERBS_ATTR_TYPE(u64), 1749 UA_MANDATORY), 1750 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN, 1751 UVERBS_ATTR_TYPE(u64), 1752 UA_MANDATORY), 1753 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, 1754 enum ib_access_flags), 1755 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, 1756 UVERBS_ATTR_TYPE(u32), 1757 UA_MANDATORY)); 1758 1759 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 1760 MLX5_IB_METHOD_DEVX_UMEM_DEREG, 1761 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE, 1762 MLX5_IB_OBJECT_DEVX_UMEM, 1763 UVERBS_ACCESS_DESTROY, 1764 UA_MANDATORY)); 1765 1766 DECLARE_UVERBS_NAMED_METHOD( 1767 MLX5_IB_METHOD_DEVX_QUERY_EQN, 1768 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC, 1769 UVERBS_ATTR_TYPE(u32), 1770 UA_MANDATORY), 1771 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN, 1772 UVERBS_ATTR_TYPE(u32), 1773 UA_MANDATORY)); 1774 1775 DECLARE_UVERBS_NAMED_METHOD( 1776 MLX5_IB_METHOD_DEVX_QUERY_UAR, 1777 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX, 1778 UVERBS_ATTR_TYPE(u32), 1779 UA_MANDATORY), 1780 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX, 1781 UVERBS_ATTR_TYPE(u32), 1782 UA_MANDATORY)); 1783 1784 DECLARE_UVERBS_NAMED_METHOD( 1785 MLX5_IB_METHOD_DEVX_OTHER, 1786 UVERBS_ATTR_PTR_IN( 1787 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN, 1788 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1789 UA_MANDATORY, 1790 UA_ALLOC_AND_COPY), 1791 UVERBS_ATTR_PTR_OUT( 1792 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, 1793 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1794 UA_MANDATORY)); 1795 1796 DECLARE_UVERBS_NAMED_METHOD( 1797 MLX5_IB_METHOD_DEVX_OBJ_CREATE, 1798 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE, 1799 MLX5_IB_OBJECT_DEVX_OBJ, 1800 UVERBS_ACCESS_NEW, 1801 UA_MANDATORY), 1802 UVERBS_ATTR_PTR_IN( 1803 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN, 1804 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1805 UA_MANDATORY, 1806 UA_ALLOC_AND_COPY), 1807 UVERBS_ATTR_PTR_OUT( 1808 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, 1809 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1810 UA_MANDATORY)); 1811 1812 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 1813 MLX5_IB_METHOD_DEVX_OBJ_DESTROY, 1814 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE, 1815 MLX5_IB_OBJECT_DEVX_OBJ, 1816 UVERBS_ACCESS_DESTROY, 1817 UA_MANDATORY)); 1818 1819 DECLARE_UVERBS_NAMED_METHOD( 1820 MLX5_IB_METHOD_DEVX_OBJ_MODIFY, 1821 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE, 1822 UVERBS_IDR_ANY_OBJECT, 1823 UVERBS_ACCESS_WRITE, 1824 UA_MANDATORY), 1825 UVERBS_ATTR_PTR_IN( 1826 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN, 1827 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1828 UA_MANDATORY, 1829 UA_ALLOC_AND_COPY), 1830 UVERBS_ATTR_PTR_OUT( 1831 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT, 1832 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1833 UA_MANDATORY)); 1834 1835 DECLARE_UVERBS_NAMED_METHOD( 1836 MLX5_IB_METHOD_DEVX_OBJ_QUERY, 1837 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE, 1838 UVERBS_IDR_ANY_OBJECT, 1839 UVERBS_ACCESS_READ, 1840 UA_MANDATORY), 1841 UVERBS_ATTR_PTR_IN( 1842 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN, 1843 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1844 UA_MANDATORY, 1845 UA_ALLOC_AND_COPY), 1846 UVERBS_ATTR_PTR_OUT( 1847 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT, 1848 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1849 UA_MANDATORY)); 1850 1851 DECLARE_UVERBS_NAMED_METHOD( 1852 MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY, 1853 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE, 1854 UVERBS_IDR_ANY_OBJECT, 1855 UVERBS_ACCESS_READ, 1856 UA_MANDATORY), 1857 UVERBS_ATTR_PTR_IN( 1858 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN, 1859 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1860 UA_MANDATORY, 1861 UA_ALLOC_AND_COPY), 1862 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN, 1863 u16, UA_MANDATORY), 1864 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD, 1865 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1866 UVERBS_ACCESS_READ, 1867 UA_MANDATORY), 1868 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID, 1869 UVERBS_ATTR_TYPE(u64), 1870 UA_MANDATORY)); 1871 1872 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX, 1873 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER), 1874 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR), 1875 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN)); 1876 1877 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ, 1878 UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup), 1879 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE), 1880 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY), 1881 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY), 1882 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY), 1883 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)); 1884 1885 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM, 1886 UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup), 1887 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG), 1888 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG)); 1889 1890 1891 DECLARE_UVERBS_NAMED_METHOD( 1892 MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC, 1893 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE, 1894 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1895 UVERBS_ACCESS_NEW, 1896 UA_MANDATORY)); 1897 1898 DECLARE_UVERBS_NAMED_OBJECT( 1899 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1900 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file), 1901 devx_hot_unplug_async_cmd_event_file, 1902 &devx_async_cmd_event_fops, "[devx_async_cmd]", 1903 O_RDONLY), 1904 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)); 1905 1906 static bool devx_is_supported(struct ib_device *device) 1907 { 1908 struct mlx5_ib_dev *dev = to_mdev(device); 1909 1910 return MLX5_CAP_GEN(dev->mdev, log_max_uctx); 1911 } 1912 1913 const struct uapi_definition mlx5_ib_devx_defs[] = { 1914 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1915 MLX5_IB_OBJECT_DEVX, 1916 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1917 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1918 MLX5_IB_OBJECT_DEVX_OBJ, 1919 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1920 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1921 MLX5_IB_OBJECT_DEVX_UMEM, 1922 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1923 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1924 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1925 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1926 {}, 1927 }; 1928