1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <rdma/ib_user_verbs.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/uverbs_types.h> 9 #include <rdma/uverbs_ioctl.h> 10 #include <rdma/mlx5_user_ioctl_cmds.h> 11 #include <rdma/mlx5_user_ioctl_verbs.h> 12 #include <rdma/ib_umem.h> 13 #include <rdma/uverbs_std_types.h> 14 #include <linux/mlx5/driver.h> 15 #include <linux/mlx5/fs.h> 16 #include "mlx5_ib.h" 17 18 #define UVERBS_MODULE_NAME mlx5_ib 19 #include <rdma/uverbs_named_ioctl.h> 20 21 enum devx_obj_flags { 22 DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0, 23 }; 24 25 struct devx_async_data { 26 struct mlx5_ib_dev *mdev; 27 struct list_head list; 28 struct ib_uobject *fd_uobj; 29 struct mlx5_async_work cb_work; 30 u16 cmd_out_len; 31 /* must be last field in this structure */ 32 struct mlx5_ib_uapi_devx_async_cmd_hdr hdr; 33 }; 34 35 #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) 36 struct devx_obj { 37 struct mlx5_core_dev *mdev; 38 u64 obj_id; 39 u32 dinlen; /* destroy inbox length */ 40 u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW]; 41 u32 flags; 42 struct mlx5_ib_devx_mr devx_mr; 43 }; 44 45 struct devx_umem { 46 struct mlx5_core_dev *mdev; 47 struct ib_umem *umem; 48 u32 page_offset; 49 int page_shift; 50 int ncont; 51 u32 dinlen; 52 u32 dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)]; 53 }; 54 55 struct devx_umem_reg_cmd { 56 void *in; 57 u32 inlen; 58 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 59 }; 60 61 static struct mlx5_ib_ucontext * 62 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs) 63 { 64 return to_mucontext(ib_uverbs_get_ucontext(attrs)); 65 } 66 67 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user) 68 { 69 u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0}; 70 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0}; 71 void *uctx; 72 int err; 73 u16 uid; 74 u32 cap = 0; 75 76 /* 0 means not supported */ 77 if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx)) 78 return -EINVAL; 79 80 uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx); 81 if (is_user && capable(CAP_NET_RAW) && 82 (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX)) 83 cap |= MLX5_UCTX_CAP_RAW_TX; 84 85 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX); 86 MLX5_SET(uctx, uctx, cap, cap); 87 88 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 89 if (err) 90 return err; 91 92 uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 93 return uid; 94 } 95 96 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) 97 { 98 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0}; 99 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0}; 100 101 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX); 102 MLX5_SET(destroy_uctx_in, in, uid, uid); 103 104 mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 105 } 106 107 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type) 108 { 109 struct devx_obj *devx_obj = obj; 110 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 111 112 switch (opcode) { 113 case MLX5_CMD_OP_DESTROY_TIR: 114 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 115 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 116 obj_id); 117 return true; 118 119 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 120 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 121 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 122 table_id); 123 return true; 124 default: 125 return false; 126 } 127 } 128 129 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id) 130 { 131 struct devx_obj *devx_obj = obj; 132 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 133 134 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 135 *counter_id = MLX5_GET(dealloc_flow_counter_in, 136 devx_obj->dinbox, 137 flow_counter_id); 138 return true; 139 } 140 141 return false; 142 } 143 144 /* 145 * As the obj_id in the firmware is not globally unique the object type 146 * must be considered upon checking for a valid object id. 147 * For that the opcode of the creator command is encoded as part of the obj_id. 148 */ 149 static u64 get_enc_obj_id(u16 opcode, u32 obj_id) 150 { 151 return ((u64)opcode << 32) | obj_id; 152 } 153 154 static u64 devx_get_obj_id(const void *in) 155 { 156 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 157 u64 obj_id; 158 159 switch (opcode) { 160 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 161 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 162 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT, 163 MLX5_GET(general_obj_in_cmd_hdr, in, 164 obj_id)); 165 break; 166 case MLX5_CMD_OP_QUERY_MKEY: 167 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY, 168 MLX5_GET(query_mkey_in, in, 169 mkey_index)); 170 break; 171 case MLX5_CMD_OP_QUERY_CQ: 172 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ, 173 MLX5_GET(query_cq_in, in, cqn)); 174 break; 175 case MLX5_CMD_OP_MODIFY_CQ: 176 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ, 177 MLX5_GET(modify_cq_in, in, cqn)); 178 break; 179 case MLX5_CMD_OP_QUERY_SQ: 180 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ, 181 MLX5_GET(query_sq_in, in, sqn)); 182 break; 183 case MLX5_CMD_OP_MODIFY_SQ: 184 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ, 185 MLX5_GET(modify_sq_in, in, sqn)); 186 break; 187 case MLX5_CMD_OP_QUERY_RQ: 188 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 189 MLX5_GET(query_rq_in, in, rqn)); 190 break; 191 case MLX5_CMD_OP_MODIFY_RQ: 192 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 193 MLX5_GET(modify_rq_in, in, rqn)); 194 break; 195 case MLX5_CMD_OP_QUERY_RMP: 196 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP, 197 MLX5_GET(query_rmp_in, in, rmpn)); 198 break; 199 case MLX5_CMD_OP_MODIFY_RMP: 200 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP, 201 MLX5_GET(modify_rmp_in, in, rmpn)); 202 break; 203 case MLX5_CMD_OP_QUERY_RQT: 204 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT, 205 MLX5_GET(query_rqt_in, in, rqtn)); 206 break; 207 case MLX5_CMD_OP_MODIFY_RQT: 208 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT, 209 MLX5_GET(modify_rqt_in, in, rqtn)); 210 break; 211 case MLX5_CMD_OP_QUERY_TIR: 212 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR, 213 MLX5_GET(query_tir_in, in, tirn)); 214 break; 215 case MLX5_CMD_OP_MODIFY_TIR: 216 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR, 217 MLX5_GET(modify_tir_in, in, tirn)); 218 break; 219 case MLX5_CMD_OP_QUERY_TIS: 220 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS, 221 MLX5_GET(query_tis_in, in, tisn)); 222 break; 223 case MLX5_CMD_OP_MODIFY_TIS: 224 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS, 225 MLX5_GET(modify_tis_in, in, tisn)); 226 break; 227 case MLX5_CMD_OP_QUERY_FLOW_TABLE: 228 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE, 229 MLX5_GET(query_flow_table_in, in, 230 table_id)); 231 break; 232 case MLX5_CMD_OP_MODIFY_FLOW_TABLE: 233 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE, 234 MLX5_GET(modify_flow_table_in, in, 235 table_id)); 236 break; 237 case MLX5_CMD_OP_QUERY_FLOW_GROUP: 238 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP, 239 MLX5_GET(query_flow_group_in, in, 240 group_id)); 241 break; 242 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: 243 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY, 244 MLX5_GET(query_fte_in, in, 245 flow_index)); 246 break; 247 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 248 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY, 249 MLX5_GET(set_fte_in, in, flow_index)); 250 break; 251 case MLX5_CMD_OP_QUERY_Q_COUNTER: 252 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER, 253 MLX5_GET(query_q_counter_in, in, 254 counter_set_id)); 255 break; 256 case MLX5_CMD_OP_QUERY_FLOW_COUNTER: 257 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER, 258 MLX5_GET(query_flow_counter_in, in, 259 flow_counter_id)); 260 break; 261 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT: 262 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT, 263 MLX5_GET(general_obj_in_cmd_hdr, in, 264 obj_id)); 265 break; 266 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT: 267 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT, 268 MLX5_GET(query_scheduling_element_in, 269 in, scheduling_element_id)); 270 break; 271 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT: 272 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT, 273 MLX5_GET(modify_scheduling_element_in, 274 in, scheduling_element_id)); 275 break; 276 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 277 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT, 278 MLX5_GET(add_vxlan_udp_dport_in, in, 279 vxlan_udp_port)); 280 break; 281 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: 282 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY, 283 MLX5_GET(query_l2_table_entry_in, in, 284 table_index)); 285 break; 286 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 287 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY, 288 MLX5_GET(set_l2_table_entry_in, in, 289 table_index)); 290 break; 291 case MLX5_CMD_OP_QUERY_QP: 292 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 293 MLX5_GET(query_qp_in, in, qpn)); 294 break; 295 case MLX5_CMD_OP_RST2INIT_QP: 296 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 297 MLX5_GET(rst2init_qp_in, in, qpn)); 298 break; 299 case MLX5_CMD_OP_INIT2RTR_QP: 300 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 301 MLX5_GET(init2rtr_qp_in, in, qpn)); 302 break; 303 case MLX5_CMD_OP_RTR2RTS_QP: 304 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 305 MLX5_GET(rtr2rts_qp_in, in, qpn)); 306 break; 307 case MLX5_CMD_OP_RTS2RTS_QP: 308 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 309 MLX5_GET(rts2rts_qp_in, in, qpn)); 310 break; 311 case MLX5_CMD_OP_SQERR2RTS_QP: 312 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 313 MLX5_GET(sqerr2rts_qp_in, in, qpn)); 314 break; 315 case MLX5_CMD_OP_2ERR_QP: 316 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 317 MLX5_GET(qp_2err_in, in, qpn)); 318 break; 319 case MLX5_CMD_OP_2RST_QP: 320 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 321 MLX5_GET(qp_2rst_in, in, qpn)); 322 break; 323 case MLX5_CMD_OP_QUERY_DCT: 324 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT, 325 MLX5_GET(query_dct_in, in, dctn)); 326 break; 327 case MLX5_CMD_OP_QUERY_XRQ: 328 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY: 329 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS: 330 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ, 331 MLX5_GET(query_xrq_in, in, xrqn)); 332 break; 333 case MLX5_CMD_OP_QUERY_XRC_SRQ: 334 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ, 335 MLX5_GET(query_xrc_srq_in, in, 336 xrc_srqn)); 337 break; 338 case MLX5_CMD_OP_ARM_XRC_SRQ: 339 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ, 340 MLX5_GET(arm_xrc_srq_in, in, xrc_srqn)); 341 break; 342 case MLX5_CMD_OP_QUERY_SRQ: 343 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ, 344 MLX5_GET(query_srq_in, in, srqn)); 345 break; 346 case MLX5_CMD_OP_ARM_RQ: 347 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 348 MLX5_GET(arm_rq_in, in, srq_number)); 349 break; 350 case MLX5_CMD_OP_DRAIN_DCT: 351 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: 352 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT, 353 MLX5_GET(drain_dct_in, in, dctn)); 354 break; 355 case MLX5_CMD_OP_ARM_XRQ: 356 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY: 357 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ, 358 MLX5_GET(arm_xrq_in, in, xrqn)); 359 break; 360 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT: 361 obj_id = get_enc_obj_id 362 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT, 363 MLX5_GET(query_packet_reformat_context_in, 364 in, packet_reformat_id)); 365 break; 366 default: 367 obj_id = 0; 368 } 369 370 return obj_id; 371 } 372 373 static bool devx_is_valid_obj_id(struct ib_uobject *uobj, const void *in) 374 { 375 u64 obj_id = devx_get_obj_id(in); 376 377 if (!obj_id) 378 return false; 379 380 switch (uobj_get_object_id(uobj)) { 381 case UVERBS_OBJECT_CQ: 382 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ, 383 to_mcq(uobj->object)->mcq.cqn) == 384 obj_id; 385 386 case UVERBS_OBJECT_SRQ: 387 { 388 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq); 389 struct mlx5_ib_dev *dev = to_mdev(uobj->context->device); 390 u16 opcode; 391 392 switch (srq->common.res) { 393 case MLX5_RES_XSRQ: 394 opcode = MLX5_CMD_OP_CREATE_XRC_SRQ; 395 break; 396 case MLX5_RES_XRQ: 397 opcode = MLX5_CMD_OP_CREATE_XRQ; 398 break; 399 default: 400 if (!dev->mdev->issi) 401 opcode = MLX5_CMD_OP_CREATE_SRQ; 402 else 403 opcode = MLX5_CMD_OP_CREATE_RMP; 404 } 405 406 return get_enc_obj_id(opcode, 407 to_msrq(uobj->object)->msrq.srqn) == 408 obj_id; 409 } 410 411 case UVERBS_OBJECT_QP: 412 { 413 struct mlx5_ib_qp *qp = to_mqp(uobj->object); 414 enum ib_qp_type qp_type = qp->ibqp.qp_type; 415 416 if (qp_type == IB_QPT_RAW_PACKET || 417 (qp->flags & MLX5_IB_QP_UNDERLAY)) { 418 struct mlx5_ib_raw_packet_qp *raw_packet_qp = 419 &qp->raw_packet_qp; 420 struct mlx5_ib_rq *rq = &raw_packet_qp->rq; 421 struct mlx5_ib_sq *sq = &raw_packet_qp->sq; 422 423 return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 424 rq->base.mqp.qpn) == obj_id || 425 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ, 426 sq->base.mqp.qpn) == obj_id || 427 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR, 428 rq->tirn) == obj_id || 429 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS, 430 sq->tisn) == obj_id); 431 } 432 433 if (qp_type == MLX5_IB_QPT_DCT) 434 return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT, 435 qp->dct.mdct.mqp.qpn) == obj_id; 436 437 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 438 qp->ibqp.qp_num) == obj_id; 439 } 440 441 case UVERBS_OBJECT_WQ: 442 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 443 to_mrwq(uobj->object)->core_qp.qpn) == 444 obj_id; 445 446 case UVERBS_OBJECT_RWQ_IND_TBL: 447 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT, 448 to_mrwq_ind_table(uobj->object)->rqtn) == 449 obj_id; 450 451 case MLX5_IB_OBJECT_DEVX_OBJ: 452 return ((struct devx_obj *)uobj->object)->obj_id == obj_id; 453 454 default: 455 return false; 456 } 457 } 458 459 static void devx_set_umem_valid(const void *in) 460 { 461 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 462 463 switch (opcode) { 464 case MLX5_CMD_OP_CREATE_MKEY: 465 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1); 466 break; 467 case MLX5_CMD_OP_CREATE_CQ: 468 { 469 void *cqc; 470 471 MLX5_SET(create_cq_in, in, cq_umem_valid, 1); 472 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 473 MLX5_SET(cqc, cqc, dbr_umem_valid, 1); 474 break; 475 } 476 case MLX5_CMD_OP_CREATE_QP: 477 { 478 void *qpc; 479 480 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 481 MLX5_SET(qpc, qpc, dbr_umem_valid, 1); 482 MLX5_SET(create_qp_in, in, wq_umem_valid, 1); 483 break; 484 } 485 486 case MLX5_CMD_OP_CREATE_RQ: 487 { 488 void *rqc, *wq; 489 490 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 491 wq = MLX5_ADDR_OF(rqc, rqc, wq); 492 MLX5_SET(wq, wq, dbr_umem_valid, 1); 493 MLX5_SET(wq, wq, wq_umem_valid, 1); 494 break; 495 } 496 497 case MLX5_CMD_OP_CREATE_SQ: 498 { 499 void *sqc, *wq; 500 501 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 502 wq = MLX5_ADDR_OF(sqc, sqc, wq); 503 MLX5_SET(wq, wq, dbr_umem_valid, 1); 504 MLX5_SET(wq, wq, wq_umem_valid, 1); 505 break; 506 } 507 508 case MLX5_CMD_OP_MODIFY_CQ: 509 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1); 510 break; 511 512 case MLX5_CMD_OP_CREATE_RMP: 513 { 514 void *rmpc, *wq; 515 516 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx); 517 wq = MLX5_ADDR_OF(rmpc, rmpc, wq); 518 MLX5_SET(wq, wq, dbr_umem_valid, 1); 519 MLX5_SET(wq, wq, wq_umem_valid, 1); 520 break; 521 } 522 523 case MLX5_CMD_OP_CREATE_XRQ: 524 { 525 void *xrqc, *wq; 526 527 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context); 528 wq = MLX5_ADDR_OF(xrqc, xrqc, wq); 529 MLX5_SET(wq, wq, dbr_umem_valid, 1); 530 MLX5_SET(wq, wq, wq_umem_valid, 1); 531 break; 532 } 533 534 case MLX5_CMD_OP_CREATE_XRC_SRQ: 535 { 536 void *xrc_srqc; 537 538 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1); 539 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in, 540 xrc_srq_context_entry); 541 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1); 542 break; 543 } 544 545 default: 546 return; 547 } 548 } 549 550 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode) 551 { 552 *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 553 554 switch (*opcode) { 555 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 556 case MLX5_CMD_OP_CREATE_MKEY: 557 case MLX5_CMD_OP_CREATE_CQ: 558 case MLX5_CMD_OP_ALLOC_PD: 559 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: 560 case MLX5_CMD_OP_CREATE_RMP: 561 case MLX5_CMD_OP_CREATE_SQ: 562 case MLX5_CMD_OP_CREATE_RQ: 563 case MLX5_CMD_OP_CREATE_RQT: 564 case MLX5_CMD_OP_CREATE_TIR: 565 case MLX5_CMD_OP_CREATE_TIS: 566 case MLX5_CMD_OP_ALLOC_Q_COUNTER: 567 case MLX5_CMD_OP_CREATE_FLOW_TABLE: 568 case MLX5_CMD_OP_CREATE_FLOW_GROUP: 569 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: 570 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT: 571 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT: 572 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT: 573 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 574 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 575 case MLX5_CMD_OP_CREATE_QP: 576 case MLX5_CMD_OP_CREATE_SRQ: 577 case MLX5_CMD_OP_CREATE_XRC_SRQ: 578 case MLX5_CMD_OP_CREATE_DCT: 579 case MLX5_CMD_OP_CREATE_XRQ: 580 case MLX5_CMD_OP_ATTACH_TO_MCG: 581 case MLX5_CMD_OP_ALLOC_XRCD: 582 return true; 583 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 584 { 585 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod); 586 if (op_mod == 0) 587 return true; 588 return false; 589 } 590 default: 591 return false; 592 } 593 } 594 595 static bool devx_is_obj_modify_cmd(const void *in) 596 { 597 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 598 599 switch (opcode) { 600 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 601 case MLX5_CMD_OP_MODIFY_CQ: 602 case MLX5_CMD_OP_MODIFY_RMP: 603 case MLX5_CMD_OP_MODIFY_SQ: 604 case MLX5_CMD_OP_MODIFY_RQ: 605 case MLX5_CMD_OP_MODIFY_RQT: 606 case MLX5_CMD_OP_MODIFY_TIR: 607 case MLX5_CMD_OP_MODIFY_TIS: 608 case MLX5_CMD_OP_MODIFY_FLOW_TABLE: 609 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT: 610 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 611 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 612 case MLX5_CMD_OP_RST2INIT_QP: 613 case MLX5_CMD_OP_INIT2RTR_QP: 614 case MLX5_CMD_OP_RTR2RTS_QP: 615 case MLX5_CMD_OP_RTS2RTS_QP: 616 case MLX5_CMD_OP_SQERR2RTS_QP: 617 case MLX5_CMD_OP_2ERR_QP: 618 case MLX5_CMD_OP_2RST_QP: 619 case MLX5_CMD_OP_ARM_XRC_SRQ: 620 case MLX5_CMD_OP_ARM_RQ: 621 case MLX5_CMD_OP_DRAIN_DCT: 622 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: 623 case MLX5_CMD_OP_ARM_XRQ: 624 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY: 625 return true; 626 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 627 { 628 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod); 629 630 if (op_mod == 1) 631 return true; 632 return false; 633 } 634 default: 635 return false; 636 } 637 } 638 639 static bool devx_is_obj_query_cmd(const void *in) 640 { 641 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 642 643 switch (opcode) { 644 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 645 case MLX5_CMD_OP_QUERY_MKEY: 646 case MLX5_CMD_OP_QUERY_CQ: 647 case MLX5_CMD_OP_QUERY_RMP: 648 case MLX5_CMD_OP_QUERY_SQ: 649 case MLX5_CMD_OP_QUERY_RQ: 650 case MLX5_CMD_OP_QUERY_RQT: 651 case MLX5_CMD_OP_QUERY_TIR: 652 case MLX5_CMD_OP_QUERY_TIS: 653 case MLX5_CMD_OP_QUERY_Q_COUNTER: 654 case MLX5_CMD_OP_QUERY_FLOW_TABLE: 655 case MLX5_CMD_OP_QUERY_FLOW_GROUP: 656 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: 657 case MLX5_CMD_OP_QUERY_FLOW_COUNTER: 658 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT: 659 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT: 660 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: 661 case MLX5_CMD_OP_QUERY_QP: 662 case MLX5_CMD_OP_QUERY_SRQ: 663 case MLX5_CMD_OP_QUERY_XRC_SRQ: 664 case MLX5_CMD_OP_QUERY_DCT: 665 case MLX5_CMD_OP_QUERY_XRQ: 666 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY: 667 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS: 668 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT: 669 return true; 670 default: 671 return false; 672 } 673 } 674 675 static bool devx_is_whitelist_cmd(void *in) 676 { 677 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 678 679 switch (opcode) { 680 case MLX5_CMD_OP_QUERY_HCA_CAP: 681 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: 682 return true; 683 default: 684 return false; 685 } 686 } 687 688 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in) 689 { 690 if (devx_is_whitelist_cmd(cmd_in)) { 691 struct mlx5_ib_dev *dev; 692 693 if (c->devx_uid) 694 return c->devx_uid; 695 696 dev = to_mdev(c->ibucontext.device); 697 if (dev->devx_whitelist_uid) 698 return dev->devx_whitelist_uid; 699 700 return -EOPNOTSUPP; 701 } 702 703 if (!c->devx_uid) 704 return -EINVAL; 705 706 return c->devx_uid; 707 } 708 static bool devx_is_general_cmd(void *in) 709 { 710 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 711 712 if (opcode >= MLX5_CMD_OP_GENERAL_START && 713 opcode < MLX5_CMD_OP_GENERAL_END) 714 return true; 715 716 switch (opcode) { 717 case MLX5_CMD_OP_QUERY_HCA_CAP: 718 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: 719 case MLX5_CMD_OP_QUERY_VPORT_STATE: 720 case MLX5_CMD_OP_QUERY_ADAPTER: 721 case MLX5_CMD_OP_QUERY_ISSI: 722 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT: 723 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS: 724 case MLX5_CMD_OP_QUERY_VNIC_ENV: 725 case MLX5_CMD_OP_QUERY_VPORT_COUNTER: 726 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG: 727 case MLX5_CMD_OP_NOP: 728 case MLX5_CMD_OP_QUERY_CONG_STATUS: 729 case MLX5_CMD_OP_QUERY_CONG_PARAMS: 730 case MLX5_CMD_OP_QUERY_CONG_STATISTICS: 731 return true; 732 default: 733 return false; 734 } 735 } 736 737 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)( 738 struct uverbs_attr_bundle *attrs) 739 { 740 struct mlx5_ib_ucontext *c; 741 struct mlx5_ib_dev *dev; 742 int user_vector; 743 int dev_eqn; 744 unsigned int irqn; 745 int err; 746 747 if (uverbs_copy_from(&user_vector, attrs, 748 MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC)) 749 return -EFAULT; 750 751 c = devx_ufile2uctx(attrs); 752 if (IS_ERR(c)) 753 return PTR_ERR(c); 754 dev = to_mdev(c->ibucontext.device); 755 756 err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn); 757 if (err < 0) 758 return err; 759 760 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN, 761 &dev_eqn, sizeof(dev_eqn))) 762 return -EFAULT; 763 764 return 0; 765 } 766 767 /* 768 *Security note: 769 * The hardware protection mechanism works like this: Each device object that 770 * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in 771 * the device specification manual) upon its creation. Then upon doorbell, 772 * hardware fetches the object context for which the doorbell was rang, and 773 * validates that the UAR through which the DB was rang matches the UAR ID 774 * of the object. 775 * If no match the doorbell is silently ignored by the hardware. Of course, 776 * the user cannot ring a doorbell on a UAR that was not mapped to it. 777 * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command 778 * mailboxes (except tagging them with UID), we expose to the user its UAR 779 * ID, so it can embed it in these objects in the expected specification 780 * format. So the only thing the user can do is hurt itself by creating a 781 * QP/SQ/CQ with a UAR ID other than his, and then in this case other users 782 * may ring a doorbell on its objects. 783 * The consequence of that will be that another user can schedule a QP/SQ 784 * of the buggy user for execution (just insert it to the hardware schedule 785 * queue or arm its CQ for event generation), no further harm is expected. 786 */ 787 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)( 788 struct uverbs_attr_bundle *attrs) 789 { 790 struct mlx5_ib_ucontext *c; 791 struct mlx5_ib_dev *dev; 792 u32 user_idx; 793 s32 dev_idx; 794 795 c = devx_ufile2uctx(attrs); 796 if (IS_ERR(c)) 797 return PTR_ERR(c); 798 dev = to_mdev(c->ibucontext.device); 799 800 if (uverbs_copy_from(&user_idx, attrs, 801 MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX)) 802 return -EFAULT; 803 804 dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true); 805 if (dev_idx < 0) 806 return dev_idx; 807 808 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX, 809 &dev_idx, sizeof(dev_idx))) 810 return -EFAULT; 811 812 return 0; 813 } 814 815 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)( 816 struct uverbs_attr_bundle *attrs) 817 { 818 struct mlx5_ib_ucontext *c; 819 struct mlx5_ib_dev *dev; 820 void *cmd_in = uverbs_attr_get_alloced_ptr( 821 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN); 822 int cmd_out_len = uverbs_attr_get_len(attrs, 823 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT); 824 void *cmd_out; 825 int err; 826 int uid; 827 828 c = devx_ufile2uctx(attrs); 829 if (IS_ERR(c)) 830 return PTR_ERR(c); 831 dev = to_mdev(c->ibucontext.device); 832 833 uid = devx_get_uid(c, cmd_in); 834 if (uid < 0) 835 return uid; 836 837 /* Only white list of some general HCA commands are allowed for this method. */ 838 if (!devx_is_general_cmd(cmd_in)) 839 return -EINVAL; 840 841 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 842 if (IS_ERR(cmd_out)) 843 return PTR_ERR(cmd_out); 844 845 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 846 err = mlx5_cmd_exec(dev->mdev, cmd_in, 847 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN), 848 cmd_out, cmd_out_len); 849 if (err) 850 return err; 851 852 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out, 853 cmd_out_len); 854 } 855 856 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din, 857 u32 *dinlen, 858 u32 *obj_id) 859 { 860 u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type); 861 u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid); 862 863 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 864 *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr); 865 866 MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id); 867 MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid); 868 869 switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) { 870 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 871 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 872 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type); 873 break; 874 875 case MLX5_CMD_OP_CREATE_UMEM: 876 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 877 MLX5_CMD_OP_DESTROY_UMEM); 878 break; 879 case MLX5_CMD_OP_CREATE_MKEY: 880 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY); 881 break; 882 case MLX5_CMD_OP_CREATE_CQ: 883 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ); 884 break; 885 case MLX5_CMD_OP_ALLOC_PD: 886 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD); 887 break; 888 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: 889 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 890 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN); 891 break; 892 case MLX5_CMD_OP_CREATE_RMP: 893 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP); 894 break; 895 case MLX5_CMD_OP_CREATE_SQ: 896 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ); 897 break; 898 case MLX5_CMD_OP_CREATE_RQ: 899 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ); 900 break; 901 case MLX5_CMD_OP_CREATE_RQT: 902 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT); 903 break; 904 case MLX5_CMD_OP_CREATE_TIR: 905 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR); 906 break; 907 case MLX5_CMD_OP_CREATE_TIS: 908 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS); 909 break; 910 case MLX5_CMD_OP_ALLOC_Q_COUNTER: 911 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 912 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 913 break; 914 case MLX5_CMD_OP_CREATE_FLOW_TABLE: 915 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in); 916 *obj_id = MLX5_GET(create_flow_table_out, out, table_id); 917 MLX5_SET(destroy_flow_table_in, din, other_vport, 918 MLX5_GET(create_flow_table_in, in, other_vport)); 919 MLX5_SET(destroy_flow_table_in, din, vport_number, 920 MLX5_GET(create_flow_table_in, in, vport_number)); 921 MLX5_SET(destroy_flow_table_in, din, table_type, 922 MLX5_GET(create_flow_table_in, in, table_type)); 923 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id); 924 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 925 MLX5_CMD_OP_DESTROY_FLOW_TABLE); 926 break; 927 case MLX5_CMD_OP_CREATE_FLOW_GROUP: 928 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in); 929 *obj_id = MLX5_GET(create_flow_group_out, out, group_id); 930 MLX5_SET(destroy_flow_group_in, din, other_vport, 931 MLX5_GET(create_flow_group_in, in, other_vport)); 932 MLX5_SET(destroy_flow_group_in, din, vport_number, 933 MLX5_GET(create_flow_group_in, in, vport_number)); 934 MLX5_SET(destroy_flow_group_in, din, table_type, 935 MLX5_GET(create_flow_group_in, in, table_type)); 936 MLX5_SET(destroy_flow_group_in, din, table_id, 937 MLX5_GET(create_flow_group_in, in, table_id)); 938 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id); 939 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 940 MLX5_CMD_OP_DESTROY_FLOW_GROUP); 941 break; 942 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 943 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in); 944 *obj_id = MLX5_GET(set_fte_in, in, flow_index); 945 MLX5_SET(delete_fte_in, din, other_vport, 946 MLX5_GET(set_fte_in, in, other_vport)); 947 MLX5_SET(delete_fte_in, din, vport_number, 948 MLX5_GET(set_fte_in, in, vport_number)); 949 MLX5_SET(delete_fte_in, din, table_type, 950 MLX5_GET(set_fte_in, in, table_type)); 951 MLX5_SET(delete_fte_in, din, table_id, 952 MLX5_GET(set_fte_in, in, table_id)); 953 MLX5_SET(delete_fte_in, din, flow_index, *obj_id); 954 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 955 MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY); 956 break; 957 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: 958 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 959 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER); 960 break; 961 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT: 962 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 963 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT); 964 break; 965 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT: 966 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 967 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT); 968 break; 969 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT: 970 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in); 971 *obj_id = MLX5_GET(create_scheduling_element_out, out, 972 scheduling_element_id); 973 MLX5_SET(destroy_scheduling_element_in, din, 974 scheduling_hierarchy, 975 MLX5_GET(create_scheduling_element_in, in, 976 scheduling_hierarchy)); 977 MLX5_SET(destroy_scheduling_element_in, din, 978 scheduling_element_id, *obj_id); 979 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 980 MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT); 981 break; 982 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 983 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in); 984 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port); 985 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id); 986 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 987 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT); 988 break; 989 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 990 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in); 991 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index); 992 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id); 993 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 994 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY); 995 break; 996 case MLX5_CMD_OP_CREATE_QP: 997 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP); 998 break; 999 case MLX5_CMD_OP_CREATE_SRQ: 1000 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ); 1001 break; 1002 case MLX5_CMD_OP_CREATE_XRC_SRQ: 1003 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 1004 MLX5_CMD_OP_DESTROY_XRC_SRQ); 1005 break; 1006 case MLX5_CMD_OP_CREATE_DCT: 1007 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT); 1008 break; 1009 case MLX5_CMD_OP_CREATE_XRQ: 1010 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ); 1011 break; 1012 case MLX5_CMD_OP_ATTACH_TO_MCG: 1013 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in); 1014 MLX5_SET(detach_from_mcg_in, din, qpn, 1015 MLX5_GET(attach_to_mcg_in, in, qpn)); 1016 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid), 1017 MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid), 1018 MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid)); 1019 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG); 1020 break; 1021 case MLX5_CMD_OP_ALLOC_XRCD: 1022 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD); 1023 break; 1024 default: 1025 /* The entry must match to one of the devx_is_obj_create_cmd */ 1026 WARN_ON(true); 1027 break; 1028 } 1029 } 1030 1031 static int devx_handle_mkey_indirect(struct devx_obj *obj, 1032 struct mlx5_ib_dev *dev, 1033 void *in, void *out) 1034 { 1035 struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table; 1036 struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr; 1037 unsigned long flags; 1038 struct mlx5_core_mkey *mkey; 1039 void *mkc; 1040 u8 key; 1041 int err; 1042 1043 mkey = &devx_mr->mmkey; 1044 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1045 key = MLX5_GET(mkc, mkc, mkey_7_0); 1046 mkey->key = mlx5_idx_to_mkey( 1047 MLX5_GET(create_mkey_out, out, mkey_index)) | key; 1048 mkey->type = MLX5_MKEY_INDIRECT_DEVX; 1049 mkey->iova = MLX5_GET64(mkc, mkc, start_addr); 1050 mkey->size = MLX5_GET64(mkc, mkc, len); 1051 mkey->pd = MLX5_GET(mkc, mkc, pd); 1052 devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size); 1053 1054 write_lock_irqsave(&table->lock, flags); 1055 err = radix_tree_insert(&table->tree, mlx5_base_mkey(mkey->key), 1056 mkey); 1057 write_unlock_irqrestore(&table->lock, flags); 1058 return err; 1059 } 1060 1061 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev, 1062 struct devx_obj *obj, 1063 void *in, int in_len) 1064 { 1065 int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) + 1066 MLX5_FLD_SZ_BYTES(create_mkey_in, 1067 memory_key_mkey_entry); 1068 void *mkc; 1069 u8 access_mode; 1070 1071 if (in_len < min_len) 1072 return -EINVAL; 1073 1074 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1075 1076 access_mode = MLX5_GET(mkc, mkc, access_mode_1_0); 1077 access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2; 1078 1079 if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS || 1080 access_mode == MLX5_MKC_ACCESS_MODE_KSM) { 1081 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 1082 obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY; 1083 return 0; 1084 } 1085 1086 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1); 1087 return 0; 1088 } 1089 1090 static void devx_free_indirect_mkey(struct rcu_head *rcu) 1091 { 1092 kfree(container_of(rcu, struct devx_obj, devx_mr.rcu)); 1093 } 1094 1095 /* This function to delete from the radix tree needs to be called before 1096 * destroying the underlying mkey. Otherwise a race might occur in case that 1097 * other thread will get the same mkey before this one will be deleted, 1098 * in that case it will fail via inserting to the tree its own data. 1099 * 1100 * Note: 1101 * An error in the destroy is not expected unless there is some other indirect 1102 * mkey which points to this one. In a kernel cleanup flow it will be just 1103 * destroyed in the iterative destruction call. In a user flow, in case 1104 * the application didn't close in the expected order it's its own problem, 1105 * the mkey won't be part of the tree, in both cases the kernel is safe. 1106 */ 1107 static void devx_cleanup_mkey(struct devx_obj *obj) 1108 { 1109 struct mlx5_mkey_table *table = &obj->mdev->priv.mkey_table; 1110 unsigned long flags; 1111 1112 write_lock_irqsave(&table->lock, flags); 1113 radix_tree_delete(&table->tree, mlx5_base_mkey(obj->devx_mr.mmkey.key)); 1114 write_unlock_irqrestore(&table->lock, flags); 1115 } 1116 1117 static int devx_obj_cleanup(struct ib_uobject *uobject, 1118 enum rdma_remove_reason why) 1119 { 1120 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 1121 struct devx_obj *obj = uobject->object; 1122 int ret; 1123 1124 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) 1125 devx_cleanup_mkey(obj); 1126 1127 ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out)); 1128 if (ib_is_destroy_retryable(ret, why, uobject)) 1129 return ret; 1130 1131 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) { 1132 struct mlx5_ib_dev *dev = to_mdev(uobject->context->device); 1133 1134 call_srcu(&dev->mr_srcu, &obj->devx_mr.rcu, 1135 devx_free_indirect_mkey); 1136 return ret; 1137 } 1138 1139 kfree(obj); 1140 return ret; 1141 } 1142 1143 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( 1144 struct uverbs_attr_bundle *attrs) 1145 { 1146 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN); 1147 int cmd_out_len = uverbs_attr_get_len(attrs, 1148 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT); 1149 int cmd_in_len = uverbs_attr_get_len(attrs, 1150 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN); 1151 void *cmd_out; 1152 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1153 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE); 1154 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1155 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1156 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); 1157 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 1158 struct devx_obj *obj; 1159 int err; 1160 int uid; 1161 u32 obj_id; 1162 u16 opcode; 1163 1164 uid = devx_get_uid(c, cmd_in); 1165 if (uid < 0) 1166 return uid; 1167 1168 if (!devx_is_obj_create_cmd(cmd_in, &opcode)) 1169 return -EINVAL; 1170 1171 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 1172 if (IS_ERR(cmd_out)) 1173 return PTR_ERR(cmd_out); 1174 1175 obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL); 1176 if (!obj) 1177 return -ENOMEM; 1178 1179 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1180 if (opcode == MLX5_CMD_OP_CREATE_MKEY) { 1181 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len); 1182 if (err) 1183 goto obj_free; 1184 } else { 1185 devx_set_umem_valid(cmd_in); 1186 } 1187 1188 err = mlx5_cmd_exec(dev->mdev, cmd_in, 1189 cmd_in_len, 1190 cmd_out, cmd_out_len); 1191 if (err) 1192 goto obj_free; 1193 1194 uobj->object = obj; 1195 obj->mdev = dev->mdev; 1196 devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen, 1197 &obj_id); 1198 WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32)); 1199 1200 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) { 1201 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out); 1202 if (err) 1203 goto obj_destroy; 1204 } 1205 1206 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len); 1207 if (err) 1208 goto err_copy; 1209 1210 obj->obj_id = get_enc_obj_id(opcode, obj_id); 1211 return 0; 1212 1213 err_copy: 1214 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) 1215 devx_cleanup_mkey(obj); 1216 obj_destroy: 1217 mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out)); 1218 obj_free: 1219 kfree(obj); 1220 return err; 1221 } 1222 1223 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)( 1224 struct uverbs_attr_bundle *attrs) 1225 { 1226 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN); 1227 int cmd_out_len = uverbs_attr_get_len(attrs, 1228 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT); 1229 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 1230 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE); 1231 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1232 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1233 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); 1234 void *cmd_out; 1235 int err; 1236 int uid; 1237 1238 uid = devx_get_uid(c, cmd_in); 1239 if (uid < 0) 1240 return uid; 1241 1242 if (!devx_is_obj_modify_cmd(cmd_in)) 1243 return -EINVAL; 1244 1245 if (!devx_is_valid_obj_id(uobj, cmd_in)) 1246 return -EINVAL; 1247 1248 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 1249 if (IS_ERR(cmd_out)) 1250 return PTR_ERR(cmd_out); 1251 1252 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1253 devx_set_umem_valid(cmd_in); 1254 1255 err = mlx5_cmd_exec(mdev->mdev, cmd_in, 1256 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN), 1257 cmd_out, cmd_out_len); 1258 if (err) 1259 return err; 1260 1261 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT, 1262 cmd_out, cmd_out_len); 1263 } 1264 1265 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)( 1266 struct uverbs_attr_bundle *attrs) 1267 { 1268 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN); 1269 int cmd_out_len = uverbs_attr_get_len(attrs, 1270 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT); 1271 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 1272 MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE); 1273 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1274 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1275 void *cmd_out; 1276 int err; 1277 int uid; 1278 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); 1279 1280 uid = devx_get_uid(c, cmd_in); 1281 if (uid < 0) 1282 return uid; 1283 1284 if (!devx_is_obj_query_cmd(cmd_in)) 1285 return -EINVAL; 1286 1287 if (!devx_is_valid_obj_id(uobj, cmd_in)) 1288 return -EINVAL; 1289 1290 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 1291 if (IS_ERR(cmd_out)) 1292 return PTR_ERR(cmd_out); 1293 1294 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1295 err = mlx5_cmd_exec(mdev->mdev, cmd_in, 1296 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN), 1297 cmd_out, cmd_out_len); 1298 if (err) 1299 return err; 1300 1301 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT, 1302 cmd_out, cmd_out_len); 1303 } 1304 1305 struct devx_async_event_queue { 1306 spinlock_t lock; 1307 wait_queue_head_t poll_wait; 1308 struct list_head event_list; 1309 atomic_t bytes_in_use; 1310 u8 is_destroyed:1; 1311 }; 1312 1313 struct devx_async_cmd_event_file { 1314 struct ib_uobject uobj; 1315 struct devx_async_event_queue ev_queue; 1316 struct mlx5_async_ctx async_ctx; 1317 }; 1318 1319 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue) 1320 { 1321 spin_lock_init(&ev_queue->lock); 1322 INIT_LIST_HEAD(&ev_queue->event_list); 1323 init_waitqueue_head(&ev_queue->poll_wait); 1324 atomic_set(&ev_queue->bytes_in_use, 0); 1325 ev_queue->is_destroyed = 0; 1326 } 1327 1328 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)( 1329 struct uverbs_attr_bundle *attrs) 1330 { 1331 struct devx_async_cmd_event_file *ev_file; 1332 1333 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1334 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE); 1335 struct mlx5_ib_dev *mdev = to_mdev(uobj->context->device); 1336 1337 ev_file = container_of(uobj, struct devx_async_cmd_event_file, 1338 uobj); 1339 devx_init_event_queue(&ev_file->ev_queue); 1340 mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx); 1341 return 0; 1342 } 1343 1344 static void devx_query_callback(int status, struct mlx5_async_work *context) 1345 { 1346 struct devx_async_data *async_data = 1347 container_of(context, struct devx_async_data, cb_work); 1348 struct ib_uobject *fd_uobj = async_data->fd_uobj; 1349 struct devx_async_cmd_event_file *ev_file; 1350 struct devx_async_event_queue *ev_queue; 1351 unsigned long flags; 1352 1353 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file, 1354 uobj); 1355 ev_queue = &ev_file->ev_queue; 1356 1357 spin_lock_irqsave(&ev_queue->lock, flags); 1358 list_add_tail(&async_data->list, &ev_queue->event_list); 1359 spin_unlock_irqrestore(&ev_queue->lock, flags); 1360 1361 wake_up_interruptible(&ev_queue->poll_wait); 1362 fput(fd_uobj->object); 1363 } 1364 1365 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */ 1366 1367 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)( 1368 struct uverbs_attr_bundle *attrs) 1369 { 1370 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, 1371 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN); 1372 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1373 attrs, 1374 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE); 1375 u16 cmd_out_len; 1376 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1377 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1378 struct ib_uobject *fd_uobj; 1379 int err; 1380 int uid; 1381 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); 1382 struct devx_async_cmd_event_file *ev_file; 1383 struct devx_async_data *async_data; 1384 1385 uid = devx_get_uid(c, cmd_in); 1386 if (uid < 0) 1387 return uid; 1388 1389 if (!devx_is_obj_query_cmd(cmd_in)) 1390 return -EINVAL; 1391 1392 err = uverbs_get_const(&cmd_out_len, attrs, 1393 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN); 1394 if (err) 1395 return err; 1396 1397 if (!devx_is_valid_obj_id(uobj, cmd_in)) 1398 return -EINVAL; 1399 1400 fd_uobj = uverbs_attr_get_uobject(attrs, 1401 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD); 1402 if (IS_ERR(fd_uobj)) 1403 return PTR_ERR(fd_uobj); 1404 1405 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file, 1406 uobj); 1407 1408 if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) > 1409 MAX_ASYNC_BYTES_IN_USE) { 1410 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use); 1411 return -EAGAIN; 1412 } 1413 1414 async_data = kvzalloc(struct_size(async_data, hdr.out_data, 1415 cmd_out_len), GFP_KERNEL); 1416 if (!async_data) { 1417 err = -ENOMEM; 1418 goto sub_bytes; 1419 } 1420 1421 err = uverbs_copy_from(&async_data->hdr.wr_id, attrs, 1422 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID); 1423 if (err) 1424 goto free_async; 1425 1426 async_data->cmd_out_len = cmd_out_len; 1427 async_data->mdev = mdev; 1428 async_data->fd_uobj = fd_uobj; 1429 1430 get_file(fd_uobj->object); 1431 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1432 err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in, 1433 uverbs_attr_get_len(attrs, 1434 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN), 1435 async_data->hdr.out_data, 1436 async_data->cmd_out_len, 1437 devx_query_callback, &async_data->cb_work); 1438 1439 if (err) 1440 goto cb_err; 1441 1442 return 0; 1443 1444 cb_err: 1445 fput(fd_uobj->object); 1446 free_async: 1447 kvfree(async_data); 1448 sub_bytes: 1449 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use); 1450 return err; 1451 } 1452 1453 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, 1454 struct uverbs_attr_bundle *attrs, 1455 struct devx_umem *obj) 1456 { 1457 u64 addr; 1458 size_t size; 1459 u32 access; 1460 int npages; 1461 int err; 1462 u32 page_mask; 1463 1464 if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) || 1465 uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN)) 1466 return -EFAULT; 1467 1468 err = uverbs_get_flags32(&access, attrs, 1469 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, 1470 IB_ACCESS_LOCAL_WRITE | 1471 IB_ACCESS_REMOTE_WRITE | 1472 IB_ACCESS_REMOTE_READ); 1473 if (err) 1474 return err; 1475 1476 err = ib_check_mr_access(access); 1477 if (err) 1478 return err; 1479 1480 obj->umem = ib_umem_get(&attrs->driver_udata, addr, size, access, 0); 1481 if (IS_ERR(obj->umem)) 1482 return PTR_ERR(obj->umem); 1483 1484 mlx5_ib_cont_pages(obj->umem, obj->umem->address, 1485 MLX5_MKEY_PAGE_SHIFT_MASK, &npages, 1486 &obj->page_shift, &obj->ncont, NULL); 1487 1488 if (!npages) { 1489 ib_umem_release(obj->umem); 1490 return -EINVAL; 1491 } 1492 1493 page_mask = (1 << obj->page_shift) - 1; 1494 obj->page_offset = obj->umem->address & page_mask; 1495 1496 return 0; 1497 } 1498 1499 static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs, 1500 struct devx_umem *obj, 1501 struct devx_umem_reg_cmd *cmd) 1502 { 1503 cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) + 1504 (MLX5_ST_SZ_BYTES(mtt) * obj->ncont); 1505 cmd->in = uverbs_zalloc(attrs, cmd->inlen); 1506 return PTR_ERR_OR_ZERO(cmd->in); 1507 } 1508 1509 static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev, 1510 struct devx_umem *obj, 1511 struct devx_umem_reg_cmd *cmd) 1512 { 1513 void *umem; 1514 __be64 *mtt; 1515 1516 umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem); 1517 mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt); 1518 1519 MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM); 1520 MLX5_SET64(umem, umem, num_of_mtt, obj->ncont); 1521 MLX5_SET(umem, umem, log_page_size, obj->page_shift - 1522 MLX5_ADAPTER_PAGE_SHIFT); 1523 MLX5_SET(umem, umem, page_offset, obj->page_offset); 1524 mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt, 1525 (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) | 1526 MLX5_IB_MTT_READ); 1527 } 1528 1529 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)( 1530 struct uverbs_attr_bundle *attrs) 1531 { 1532 struct devx_umem_reg_cmd cmd; 1533 struct devx_umem *obj; 1534 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1535 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE); 1536 u32 obj_id; 1537 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1538 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1539 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); 1540 int err; 1541 1542 if (!c->devx_uid) 1543 return -EINVAL; 1544 1545 obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL); 1546 if (!obj) 1547 return -ENOMEM; 1548 1549 err = devx_umem_get(dev, &c->ibucontext, attrs, obj); 1550 if (err) 1551 goto err_obj_free; 1552 1553 err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd); 1554 if (err) 1555 goto err_umem_release; 1556 1557 devx_umem_reg_cmd_build(dev, obj, &cmd); 1558 1559 MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid); 1560 err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out, 1561 sizeof(cmd.out)); 1562 if (err) 1563 goto err_umem_release; 1564 1565 obj->mdev = dev->mdev; 1566 uobj->object = obj; 1567 devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id); 1568 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id)); 1569 if (err) 1570 goto err_umem_destroy; 1571 1572 return 0; 1573 1574 err_umem_destroy: 1575 mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out)); 1576 err_umem_release: 1577 ib_umem_release(obj->umem); 1578 err_obj_free: 1579 kfree(obj); 1580 return err; 1581 } 1582 1583 static int devx_umem_cleanup(struct ib_uobject *uobject, 1584 enum rdma_remove_reason why) 1585 { 1586 struct devx_umem *obj = uobject->object; 1587 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 1588 int err; 1589 1590 err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out)); 1591 if (ib_is_destroy_retryable(err, why, uobject)) 1592 return err; 1593 1594 ib_umem_release(obj->umem); 1595 kfree(obj); 1596 return 0; 1597 } 1598 1599 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf, 1600 size_t count, loff_t *pos) 1601 { 1602 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data; 1603 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; 1604 struct devx_async_data *event; 1605 int ret = 0; 1606 size_t eventsz; 1607 1608 spin_lock_irq(&ev_queue->lock); 1609 1610 while (list_empty(&ev_queue->event_list)) { 1611 spin_unlock_irq(&ev_queue->lock); 1612 1613 if (filp->f_flags & O_NONBLOCK) 1614 return -EAGAIN; 1615 1616 if (wait_event_interruptible( 1617 ev_queue->poll_wait, 1618 (!list_empty(&ev_queue->event_list) || 1619 ev_queue->is_destroyed))) { 1620 return -ERESTARTSYS; 1621 } 1622 1623 if (list_empty(&ev_queue->event_list) && 1624 ev_queue->is_destroyed) 1625 return -EIO; 1626 1627 spin_lock_irq(&ev_queue->lock); 1628 } 1629 1630 event = list_entry(ev_queue->event_list.next, 1631 struct devx_async_data, list); 1632 eventsz = event->cmd_out_len + 1633 sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr); 1634 1635 if (eventsz > count) { 1636 spin_unlock_irq(&ev_queue->lock); 1637 return -ENOSPC; 1638 } 1639 1640 list_del(ev_queue->event_list.next); 1641 spin_unlock_irq(&ev_queue->lock); 1642 1643 if (copy_to_user(buf, &event->hdr, eventsz)) 1644 ret = -EFAULT; 1645 else 1646 ret = eventsz; 1647 1648 atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use); 1649 kvfree(event); 1650 return ret; 1651 } 1652 1653 static int devx_async_cmd_event_close(struct inode *inode, struct file *filp) 1654 { 1655 struct ib_uobject *uobj = filp->private_data; 1656 struct devx_async_cmd_event_file *comp_ev_file = container_of( 1657 uobj, struct devx_async_cmd_event_file, uobj); 1658 struct devx_async_data *entry, *tmp; 1659 1660 spin_lock_irq(&comp_ev_file->ev_queue.lock); 1661 list_for_each_entry_safe(entry, tmp, 1662 &comp_ev_file->ev_queue.event_list, list) 1663 kvfree(entry); 1664 spin_unlock_irq(&comp_ev_file->ev_queue.lock); 1665 1666 uverbs_close_fd(filp); 1667 return 0; 1668 } 1669 1670 static __poll_t devx_async_cmd_event_poll(struct file *filp, 1671 struct poll_table_struct *wait) 1672 { 1673 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data; 1674 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; 1675 __poll_t pollflags = 0; 1676 1677 poll_wait(filp, &ev_queue->poll_wait, wait); 1678 1679 spin_lock_irq(&ev_queue->lock); 1680 if (ev_queue->is_destroyed) 1681 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 1682 else if (!list_empty(&ev_queue->event_list)) 1683 pollflags = EPOLLIN | EPOLLRDNORM; 1684 spin_unlock_irq(&ev_queue->lock); 1685 1686 return pollflags; 1687 } 1688 1689 const struct file_operations devx_async_cmd_event_fops = { 1690 .owner = THIS_MODULE, 1691 .read = devx_async_cmd_event_read, 1692 .poll = devx_async_cmd_event_poll, 1693 .release = devx_async_cmd_event_close, 1694 .llseek = no_llseek, 1695 }; 1696 1697 static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj, 1698 enum rdma_remove_reason why) 1699 { 1700 struct devx_async_cmd_event_file *comp_ev_file = 1701 container_of(uobj, struct devx_async_cmd_event_file, 1702 uobj); 1703 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; 1704 1705 spin_lock_irq(&ev_queue->lock); 1706 ev_queue->is_destroyed = 1; 1707 spin_unlock_irq(&ev_queue->lock); 1708 1709 if (why == RDMA_REMOVE_DRIVER_REMOVE) 1710 wake_up_interruptible(&ev_queue->poll_wait); 1711 1712 mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx); 1713 return 0; 1714 }; 1715 1716 DECLARE_UVERBS_NAMED_METHOD( 1717 MLX5_IB_METHOD_DEVX_UMEM_REG, 1718 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE, 1719 MLX5_IB_OBJECT_DEVX_UMEM, 1720 UVERBS_ACCESS_NEW, 1721 UA_MANDATORY), 1722 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR, 1723 UVERBS_ATTR_TYPE(u64), 1724 UA_MANDATORY), 1725 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN, 1726 UVERBS_ATTR_TYPE(u64), 1727 UA_MANDATORY), 1728 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, 1729 enum ib_access_flags), 1730 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, 1731 UVERBS_ATTR_TYPE(u32), 1732 UA_MANDATORY)); 1733 1734 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 1735 MLX5_IB_METHOD_DEVX_UMEM_DEREG, 1736 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE, 1737 MLX5_IB_OBJECT_DEVX_UMEM, 1738 UVERBS_ACCESS_DESTROY, 1739 UA_MANDATORY)); 1740 1741 DECLARE_UVERBS_NAMED_METHOD( 1742 MLX5_IB_METHOD_DEVX_QUERY_EQN, 1743 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC, 1744 UVERBS_ATTR_TYPE(u32), 1745 UA_MANDATORY), 1746 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN, 1747 UVERBS_ATTR_TYPE(u32), 1748 UA_MANDATORY)); 1749 1750 DECLARE_UVERBS_NAMED_METHOD( 1751 MLX5_IB_METHOD_DEVX_QUERY_UAR, 1752 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX, 1753 UVERBS_ATTR_TYPE(u32), 1754 UA_MANDATORY), 1755 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX, 1756 UVERBS_ATTR_TYPE(u32), 1757 UA_MANDATORY)); 1758 1759 DECLARE_UVERBS_NAMED_METHOD( 1760 MLX5_IB_METHOD_DEVX_OTHER, 1761 UVERBS_ATTR_PTR_IN( 1762 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN, 1763 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1764 UA_MANDATORY, 1765 UA_ALLOC_AND_COPY), 1766 UVERBS_ATTR_PTR_OUT( 1767 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, 1768 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1769 UA_MANDATORY)); 1770 1771 DECLARE_UVERBS_NAMED_METHOD( 1772 MLX5_IB_METHOD_DEVX_OBJ_CREATE, 1773 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE, 1774 MLX5_IB_OBJECT_DEVX_OBJ, 1775 UVERBS_ACCESS_NEW, 1776 UA_MANDATORY), 1777 UVERBS_ATTR_PTR_IN( 1778 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN, 1779 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1780 UA_MANDATORY, 1781 UA_ALLOC_AND_COPY), 1782 UVERBS_ATTR_PTR_OUT( 1783 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, 1784 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1785 UA_MANDATORY)); 1786 1787 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 1788 MLX5_IB_METHOD_DEVX_OBJ_DESTROY, 1789 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE, 1790 MLX5_IB_OBJECT_DEVX_OBJ, 1791 UVERBS_ACCESS_DESTROY, 1792 UA_MANDATORY)); 1793 1794 DECLARE_UVERBS_NAMED_METHOD( 1795 MLX5_IB_METHOD_DEVX_OBJ_MODIFY, 1796 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE, 1797 UVERBS_IDR_ANY_OBJECT, 1798 UVERBS_ACCESS_WRITE, 1799 UA_MANDATORY), 1800 UVERBS_ATTR_PTR_IN( 1801 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN, 1802 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1803 UA_MANDATORY, 1804 UA_ALLOC_AND_COPY), 1805 UVERBS_ATTR_PTR_OUT( 1806 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT, 1807 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1808 UA_MANDATORY)); 1809 1810 DECLARE_UVERBS_NAMED_METHOD( 1811 MLX5_IB_METHOD_DEVX_OBJ_QUERY, 1812 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE, 1813 UVERBS_IDR_ANY_OBJECT, 1814 UVERBS_ACCESS_READ, 1815 UA_MANDATORY), 1816 UVERBS_ATTR_PTR_IN( 1817 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN, 1818 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1819 UA_MANDATORY, 1820 UA_ALLOC_AND_COPY), 1821 UVERBS_ATTR_PTR_OUT( 1822 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT, 1823 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1824 UA_MANDATORY)); 1825 1826 DECLARE_UVERBS_NAMED_METHOD( 1827 MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY, 1828 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE, 1829 UVERBS_IDR_ANY_OBJECT, 1830 UVERBS_ACCESS_READ, 1831 UA_MANDATORY), 1832 UVERBS_ATTR_PTR_IN( 1833 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN, 1834 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1835 UA_MANDATORY, 1836 UA_ALLOC_AND_COPY), 1837 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN, 1838 u16, UA_MANDATORY), 1839 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD, 1840 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1841 UVERBS_ACCESS_READ, 1842 UA_MANDATORY), 1843 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID, 1844 UVERBS_ATTR_TYPE(u64), 1845 UA_MANDATORY)); 1846 1847 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX, 1848 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER), 1849 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR), 1850 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN)); 1851 1852 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ, 1853 UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup), 1854 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE), 1855 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY), 1856 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY), 1857 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY), 1858 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)); 1859 1860 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM, 1861 UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup), 1862 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG), 1863 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG)); 1864 1865 1866 DECLARE_UVERBS_NAMED_METHOD( 1867 MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC, 1868 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE, 1869 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1870 UVERBS_ACCESS_NEW, 1871 UA_MANDATORY)); 1872 1873 DECLARE_UVERBS_NAMED_OBJECT( 1874 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1875 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file), 1876 devx_hot_unplug_async_cmd_event_file, 1877 &devx_async_cmd_event_fops, "[devx_async_cmd]", 1878 O_RDONLY), 1879 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)); 1880 1881 static bool devx_is_supported(struct ib_device *device) 1882 { 1883 struct mlx5_ib_dev *dev = to_mdev(device); 1884 1885 return !dev->rep && MLX5_CAP_GEN(dev->mdev, log_max_uctx); 1886 } 1887 1888 const struct uapi_definition mlx5_ib_devx_defs[] = { 1889 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1890 MLX5_IB_OBJECT_DEVX, 1891 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1892 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1893 MLX5_IB_OBJECT_DEVX_OBJ, 1894 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1895 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1896 MLX5_IB_OBJECT_DEVX_UMEM, 1897 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1898 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1899 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1900 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1901 {}, 1902 }; 1903