1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <rdma/ib_user_verbs.h> 7 #include <rdma/ib_verbs.h> 8 #include <rdma/uverbs_types.h> 9 #include <rdma/uverbs_ioctl.h> 10 #include <rdma/mlx5_user_ioctl_cmds.h> 11 #include <rdma/mlx5_user_ioctl_verbs.h> 12 #include <rdma/ib_umem.h> 13 #include <rdma/uverbs_std_types.h> 14 #include <linux/mlx5/driver.h> 15 #include <linux/mlx5/fs.h> 16 #include "mlx5_ib.h" 17 18 #define UVERBS_MODULE_NAME mlx5_ib 19 #include <rdma/uverbs_named_ioctl.h> 20 21 enum devx_obj_flags { 22 DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0, 23 DEVX_OBJ_FLAGS_DCT = 1 << 1, 24 }; 25 26 struct devx_async_data { 27 struct mlx5_ib_dev *mdev; 28 struct list_head list; 29 struct ib_uobject *fd_uobj; 30 struct mlx5_async_work cb_work; 31 u16 cmd_out_len; 32 /* must be last field in this structure */ 33 struct mlx5_ib_uapi_devx_async_cmd_hdr hdr; 34 }; 35 36 #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) 37 struct devx_obj { 38 struct mlx5_core_dev *mdev; 39 u64 obj_id; 40 u32 dinlen; /* destroy inbox length */ 41 u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW]; 42 u32 flags; 43 union { 44 struct mlx5_ib_devx_mr devx_mr; 45 struct mlx5_core_dct core_dct; 46 }; 47 }; 48 49 struct devx_umem { 50 struct mlx5_core_dev *mdev; 51 struct ib_umem *umem; 52 u32 page_offset; 53 int page_shift; 54 int ncont; 55 u32 dinlen; 56 u32 dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)]; 57 }; 58 59 struct devx_umem_reg_cmd { 60 void *in; 61 u32 inlen; 62 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 63 }; 64 65 static struct mlx5_ib_ucontext * 66 devx_ufile2uctx(const struct uverbs_attr_bundle *attrs) 67 { 68 return to_mucontext(ib_uverbs_get_ucontext(attrs)); 69 } 70 71 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user) 72 { 73 u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0}; 74 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0}; 75 void *uctx; 76 int err; 77 u16 uid; 78 u32 cap = 0; 79 80 /* 0 means not supported */ 81 if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx)) 82 return -EINVAL; 83 84 uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx); 85 if (is_user && capable(CAP_NET_RAW) && 86 (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX)) 87 cap |= MLX5_UCTX_CAP_RAW_TX; 88 if (is_user && capable(CAP_SYS_RAWIO) && 89 (MLX5_CAP_GEN(dev->mdev, uctx_cap) & 90 MLX5_UCTX_CAP_INTERNAL_DEV_RES)) 91 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES; 92 93 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX); 94 MLX5_SET(uctx, uctx, cap, cap); 95 96 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 97 if (err) 98 return err; 99 100 uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 101 return uid; 102 } 103 104 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) 105 { 106 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0}; 107 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0}; 108 109 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX); 110 MLX5_SET(destroy_uctx_in, in, uid, uid); 111 112 mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 113 } 114 115 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type) 116 { 117 struct devx_obj *devx_obj = obj; 118 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 119 120 switch (opcode) { 121 case MLX5_CMD_OP_DESTROY_TIR: 122 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 123 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 124 obj_id); 125 return true; 126 127 case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 128 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 129 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 130 table_id); 131 return true; 132 default: 133 return false; 134 } 135 } 136 137 bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id) 138 { 139 struct devx_obj *devx_obj = obj; 140 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 141 142 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 143 *counter_id = MLX5_GET(dealloc_flow_counter_in, 144 devx_obj->dinbox, 145 flow_counter_id); 146 return true; 147 } 148 149 return false; 150 } 151 152 /* 153 * As the obj_id in the firmware is not globally unique the object type 154 * must be considered upon checking for a valid object id. 155 * For that the opcode of the creator command is encoded as part of the obj_id. 156 */ 157 static u64 get_enc_obj_id(u16 opcode, u32 obj_id) 158 { 159 return ((u64)opcode << 32) | obj_id; 160 } 161 162 static u64 devx_get_obj_id(const void *in) 163 { 164 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 165 u64 obj_id; 166 167 switch (opcode) { 168 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 169 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 170 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT, 171 MLX5_GET(general_obj_in_cmd_hdr, in, 172 obj_id)); 173 break; 174 case MLX5_CMD_OP_QUERY_MKEY: 175 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY, 176 MLX5_GET(query_mkey_in, in, 177 mkey_index)); 178 break; 179 case MLX5_CMD_OP_QUERY_CQ: 180 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ, 181 MLX5_GET(query_cq_in, in, cqn)); 182 break; 183 case MLX5_CMD_OP_MODIFY_CQ: 184 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ, 185 MLX5_GET(modify_cq_in, in, cqn)); 186 break; 187 case MLX5_CMD_OP_QUERY_SQ: 188 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ, 189 MLX5_GET(query_sq_in, in, sqn)); 190 break; 191 case MLX5_CMD_OP_MODIFY_SQ: 192 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ, 193 MLX5_GET(modify_sq_in, in, sqn)); 194 break; 195 case MLX5_CMD_OP_QUERY_RQ: 196 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 197 MLX5_GET(query_rq_in, in, rqn)); 198 break; 199 case MLX5_CMD_OP_MODIFY_RQ: 200 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 201 MLX5_GET(modify_rq_in, in, rqn)); 202 break; 203 case MLX5_CMD_OP_QUERY_RMP: 204 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP, 205 MLX5_GET(query_rmp_in, in, rmpn)); 206 break; 207 case MLX5_CMD_OP_MODIFY_RMP: 208 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP, 209 MLX5_GET(modify_rmp_in, in, rmpn)); 210 break; 211 case MLX5_CMD_OP_QUERY_RQT: 212 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT, 213 MLX5_GET(query_rqt_in, in, rqtn)); 214 break; 215 case MLX5_CMD_OP_MODIFY_RQT: 216 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT, 217 MLX5_GET(modify_rqt_in, in, rqtn)); 218 break; 219 case MLX5_CMD_OP_QUERY_TIR: 220 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR, 221 MLX5_GET(query_tir_in, in, tirn)); 222 break; 223 case MLX5_CMD_OP_MODIFY_TIR: 224 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR, 225 MLX5_GET(modify_tir_in, in, tirn)); 226 break; 227 case MLX5_CMD_OP_QUERY_TIS: 228 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS, 229 MLX5_GET(query_tis_in, in, tisn)); 230 break; 231 case MLX5_CMD_OP_MODIFY_TIS: 232 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS, 233 MLX5_GET(modify_tis_in, in, tisn)); 234 break; 235 case MLX5_CMD_OP_QUERY_FLOW_TABLE: 236 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE, 237 MLX5_GET(query_flow_table_in, in, 238 table_id)); 239 break; 240 case MLX5_CMD_OP_MODIFY_FLOW_TABLE: 241 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE, 242 MLX5_GET(modify_flow_table_in, in, 243 table_id)); 244 break; 245 case MLX5_CMD_OP_QUERY_FLOW_GROUP: 246 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP, 247 MLX5_GET(query_flow_group_in, in, 248 group_id)); 249 break; 250 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: 251 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY, 252 MLX5_GET(query_fte_in, in, 253 flow_index)); 254 break; 255 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 256 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY, 257 MLX5_GET(set_fte_in, in, flow_index)); 258 break; 259 case MLX5_CMD_OP_QUERY_Q_COUNTER: 260 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER, 261 MLX5_GET(query_q_counter_in, in, 262 counter_set_id)); 263 break; 264 case MLX5_CMD_OP_QUERY_FLOW_COUNTER: 265 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER, 266 MLX5_GET(query_flow_counter_in, in, 267 flow_counter_id)); 268 break; 269 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT: 270 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT, 271 MLX5_GET(general_obj_in_cmd_hdr, in, 272 obj_id)); 273 break; 274 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT: 275 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT, 276 MLX5_GET(query_scheduling_element_in, 277 in, scheduling_element_id)); 278 break; 279 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT: 280 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT, 281 MLX5_GET(modify_scheduling_element_in, 282 in, scheduling_element_id)); 283 break; 284 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 285 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT, 286 MLX5_GET(add_vxlan_udp_dport_in, in, 287 vxlan_udp_port)); 288 break; 289 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: 290 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY, 291 MLX5_GET(query_l2_table_entry_in, in, 292 table_index)); 293 break; 294 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 295 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY, 296 MLX5_GET(set_l2_table_entry_in, in, 297 table_index)); 298 break; 299 case MLX5_CMD_OP_QUERY_QP: 300 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 301 MLX5_GET(query_qp_in, in, qpn)); 302 break; 303 case MLX5_CMD_OP_RST2INIT_QP: 304 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 305 MLX5_GET(rst2init_qp_in, in, qpn)); 306 break; 307 case MLX5_CMD_OP_INIT2RTR_QP: 308 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 309 MLX5_GET(init2rtr_qp_in, in, qpn)); 310 break; 311 case MLX5_CMD_OP_RTR2RTS_QP: 312 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 313 MLX5_GET(rtr2rts_qp_in, in, qpn)); 314 break; 315 case MLX5_CMD_OP_RTS2RTS_QP: 316 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 317 MLX5_GET(rts2rts_qp_in, in, qpn)); 318 break; 319 case MLX5_CMD_OP_SQERR2RTS_QP: 320 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 321 MLX5_GET(sqerr2rts_qp_in, in, qpn)); 322 break; 323 case MLX5_CMD_OP_2ERR_QP: 324 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 325 MLX5_GET(qp_2err_in, in, qpn)); 326 break; 327 case MLX5_CMD_OP_2RST_QP: 328 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 329 MLX5_GET(qp_2rst_in, in, qpn)); 330 break; 331 case MLX5_CMD_OP_QUERY_DCT: 332 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT, 333 MLX5_GET(query_dct_in, in, dctn)); 334 break; 335 case MLX5_CMD_OP_QUERY_XRQ: 336 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY: 337 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS: 338 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ, 339 MLX5_GET(query_xrq_in, in, xrqn)); 340 break; 341 case MLX5_CMD_OP_QUERY_XRC_SRQ: 342 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ, 343 MLX5_GET(query_xrc_srq_in, in, 344 xrc_srqn)); 345 break; 346 case MLX5_CMD_OP_ARM_XRC_SRQ: 347 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ, 348 MLX5_GET(arm_xrc_srq_in, in, xrc_srqn)); 349 break; 350 case MLX5_CMD_OP_QUERY_SRQ: 351 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ, 352 MLX5_GET(query_srq_in, in, srqn)); 353 break; 354 case MLX5_CMD_OP_ARM_RQ: 355 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 356 MLX5_GET(arm_rq_in, in, srq_number)); 357 break; 358 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: 359 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT, 360 MLX5_GET(drain_dct_in, in, dctn)); 361 break; 362 case MLX5_CMD_OP_ARM_XRQ: 363 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY: 364 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ, 365 MLX5_GET(arm_xrq_in, in, xrqn)); 366 break; 367 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT: 368 obj_id = get_enc_obj_id 369 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT, 370 MLX5_GET(query_packet_reformat_context_in, 371 in, packet_reformat_id)); 372 break; 373 default: 374 obj_id = 0; 375 } 376 377 return obj_id; 378 } 379 380 static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs, 381 struct ib_uobject *uobj, const void *in) 382 { 383 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 384 u64 obj_id = devx_get_obj_id(in); 385 386 if (!obj_id) 387 return false; 388 389 switch (uobj_get_object_id(uobj)) { 390 case UVERBS_OBJECT_CQ: 391 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ, 392 to_mcq(uobj->object)->mcq.cqn) == 393 obj_id; 394 395 case UVERBS_OBJECT_SRQ: 396 { 397 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq); 398 u16 opcode; 399 400 switch (srq->common.res) { 401 case MLX5_RES_XSRQ: 402 opcode = MLX5_CMD_OP_CREATE_XRC_SRQ; 403 break; 404 case MLX5_RES_XRQ: 405 opcode = MLX5_CMD_OP_CREATE_XRQ; 406 break; 407 default: 408 if (!dev->mdev->issi) 409 opcode = MLX5_CMD_OP_CREATE_SRQ; 410 else 411 opcode = MLX5_CMD_OP_CREATE_RMP; 412 } 413 414 return get_enc_obj_id(opcode, 415 to_msrq(uobj->object)->msrq.srqn) == 416 obj_id; 417 } 418 419 case UVERBS_OBJECT_QP: 420 { 421 struct mlx5_ib_qp *qp = to_mqp(uobj->object); 422 enum ib_qp_type qp_type = qp->ibqp.qp_type; 423 424 if (qp_type == IB_QPT_RAW_PACKET || 425 (qp->flags & MLX5_IB_QP_UNDERLAY)) { 426 struct mlx5_ib_raw_packet_qp *raw_packet_qp = 427 &qp->raw_packet_qp; 428 struct mlx5_ib_rq *rq = &raw_packet_qp->rq; 429 struct mlx5_ib_sq *sq = &raw_packet_qp->sq; 430 431 return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 432 rq->base.mqp.qpn) == obj_id || 433 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ, 434 sq->base.mqp.qpn) == obj_id || 435 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR, 436 rq->tirn) == obj_id || 437 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS, 438 sq->tisn) == obj_id); 439 } 440 441 if (qp_type == MLX5_IB_QPT_DCT) 442 return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT, 443 qp->dct.mdct.mqp.qpn) == obj_id; 444 445 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP, 446 qp->ibqp.qp_num) == obj_id; 447 } 448 449 case UVERBS_OBJECT_WQ: 450 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ, 451 to_mrwq(uobj->object)->core_qp.qpn) == 452 obj_id; 453 454 case UVERBS_OBJECT_RWQ_IND_TBL: 455 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT, 456 to_mrwq_ind_table(uobj->object)->rqtn) == 457 obj_id; 458 459 case MLX5_IB_OBJECT_DEVX_OBJ: 460 return ((struct devx_obj *)uobj->object)->obj_id == obj_id; 461 462 default: 463 return false; 464 } 465 } 466 467 static void devx_set_umem_valid(const void *in) 468 { 469 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 470 471 switch (opcode) { 472 case MLX5_CMD_OP_CREATE_MKEY: 473 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1); 474 break; 475 case MLX5_CMD_OP_CREATE_CQ: 476 { 477 void *cqc; 478 479 MLX5_SET(create_cq_in, in, cq_umem_valid, 1); 480 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 481 MLX5_SET(cqc, cqc, dbr_umem_valid, 1); 482 break; 483 } 484 case MLX5_CMD_OP_CREATE_QP: 485 { 486 void *qpc; 487 488 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 489 MLX5_SET(qpc, qpc, dbr_umem_valid, 1); 490 MLX5_SET(create_qp_in, in, wq_umem_valid, 1); 491 break; 492 } 493 494 case MLX5_CMD_OP_CREATE_RQ: 495 { 496 void *rqc, *wq; 497 498 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 499 wq = MLX5_ADDR_OF(rqc, rqc, wq); 500 MLX5_SET(wq, wq, dbr_umem_valid, 1); 501 MLX5_SET(wq, wq, wq_umem_valid, 1); 502 break; 503 } 504 505 case MLX5_CMD_OP_CREATE_SQ: 506 { 507 void *sqc, *wq; 508 509 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 510 wq = MLX5_ADDR_OF(sqc, sqc, wq); 511 MLX5_SET(wq, wq, dbr_umem_valid, 1); 512 MLX5_SET(wq, wq, wq_umem_valid, 1); 513 break; 514 } 515 516 case MLX5_CMD_OP_MODIFY_CQ: 517 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1); 518 break; 519 520 case MLX5_CMD_OP_CREATE_RMP: 521 { 522 void *rmpc, *wq; 523 524 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx); 525 wq = MLX5_ADDR_OF(rmpc, rmpc, wq); 526 MLX5_SET(wq, wq, dbr_umem_valid, 1); 527 MLX5_SET(wq, wq, wq_umem_valid, 1); 528 break; 529 } 530 531 case MLX5_CMD_OP_CREATE_XRQ: 532 { 533 void *xrqc, *wq; 534 535 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context); 536 wq = MLX5_ADDR_OF(xrqc, xrqc, wq); 537 MLX5_SET(wq, wq, dbr_umem_valid, 1); 538 MLX5_SET(wq, wq, wq_umem_valid, 1); 539 break; 540 } 541 542 case MLX5_CMD_OP_CREATE_XRC_SRQ: 543 { 544 void *xrc_srqc; 545 546 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1); 547 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in, 548 xrc_srq_context_entry); 549 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1); 550 break; 551 } 552 553 default: 554 return; 555 } 556 } 557 558 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode) 559 { 560 *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 561 562 switch (*opcode) { 563 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 564 case MLX5_CMD_OP_CREATE_MKEY: 565 case MLX5_CMD_OP_CREATE_CQ: 566 case MLX5_CMD_OP_ALLOC_PD: 567 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: 568 case MLX5_CMD_OP_CREATE_RMP: 569 case MLX5_CMD_OP_CREATE_SQ: 570 case MLX5_CMD_OP_CREATE_RQ: 571 case MLX5_CMD_OP_CREATE_RQT: 572 case MLX5_CMD_OP_CREATE_TIR: 573 case MLX5_CMD_OP_CREATE_TIS: 574 case MLX5_CMD_OP_ALLOC_Q_COUNTER: 575 case MLX5_CMD_OP_CREATE_FLOW_TABLE: 576 case MLX5_CMD_OP_CREATE_FLOW_GROUP: 577 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: 578 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT: 579 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT: 580 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT: 581 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 582 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 583 case MLX5_CMD_OP_CREATE_QP: 584 case MLX5_CMD_OP_CREATE_SRQ: 585 case MLX5_CMD_OP_CREATE_XRC_SRQ: 586 case MLX5_CMD_OP_CREATE_DCT: 587 case MLX5_CMD_OP_CREATE_XRQ: 588 case MLX5_CMD_OP_ATTACH_TO_MCG: 589 case MLX5_CMD_OP_ALLOC_XRCD: 590 return true; 591 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 592 { 593 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod); 594 if (op_mod == 0) 595 return true; 596 return false; 597 } 598 default: 599 return false; 600 } 601 } 602 603 static bool devx_is_obj_modify_cmd(const void *in) 604 { 605 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 606 607 switch (opcode) { 608 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: 609 case MLX5_CMD_OP_MODIFY_CQ: 610 case MLX5_CMD_OP_MODIFY_RMP: 611 case MLX5_CMD_OP_MODIFY_SQ: 612 case MLX5_CMD_OP_MODIFY_RQ: 613 case MLX5_CMD_OP_MODIFY_RQT: 614 case MLX5_CMD_OP_MODIFY_TIR: 615 case MLX5_CMD_OP_MODIFY_TIS: 616 case MLX5_CMD_OP_MODIFY_FLOW_TABLE: 617 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT: 618 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 619 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 620 case MLX5_CMD_OP_RST2INIT_QP: 621 case MLX5_CMD_OP_INIT2RTR_QP: 622 case MLX5_CMD_OP_RTR2RTS_QP: 623 case MLX5_CMD_OP_RTS2RTS_QP: 624 case MLX5_CMD_OP_SQERR2RTS_QP: 625 case MLX5_CMD_OP_2ERR_QP: 626 case MLX5_CMD_OP_2RST_QP: 627 case MLX5_CMD_OP_ARM_XRC_SRQ: 628 case MLX5_CMD_OP_ARM_RQ: 629 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION: 630 case MLX5_CMD_OP_ARM_XRQ: 631 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY: 632 return true; 633 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 634 { 635 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod); 636 637 if (op_mod == 1) 638 return true; 639 return false; 640 } 641 default: 642 return false; 643 } 644 } 645 646 static bool devx_is_obj_query_cmd(const void *in) 647 { 648 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 649 650 switch (opcode) { 651 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: 652 case MLX5_CMD_OP_QUERY_MKEY: 653 case MLX5_CMD_OP_QUERY_CQ: 654 case MLX5_CMD_OP_QUERY_RMP: 655 case MLX5_CMD_OP_QUERY_SQ: 656 case MLX5_CMD_OP_QUERY_RQ: 657 case MLX5_CMD_OP_QUERY_RQT: 658 case MLX5_CMD_OP_QUERY_TIR: 659 case MLX5_CMD_OP_QUERY_TIS: 660 case MLX5_CMD_OP_QUERY_Q_COUNTER: 661 case MLX5_CMD_OP_QUERY_FLOW_TABLE: 662 case MLX5_CMD_OP_QUERY_FLOW_GROUP: 663 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY: 664 case MLX5_CMD_OP_QUERY_FLOW_COUNTER: 665 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT: 666 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT: 667 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY: 668 case MLX5_CMD_OP_QUERY_QP: 669 case MLX5_CMD_OP_QUERY_SRQ: 670 case MLX5_CMD_OP_QUERY_XRC_SRQ: 671 case MLX5_CMD_OP_QUERY_DCT: 672 case MLX5_CMD_OP_QUERY_XRQ: 673 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY: 674 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS: 675 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT: 676 return true; 677 default: 678 return false; 679 } 680 } 681 682 static bool devx_is_whitelist_cmd(void *in) 683 { 684 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 685 686 switch (opcode) { 687 case MLX5_CMD_OP_QUERY_HCA_CAP: 688 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: 689 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT: 690 return true; 691 default: 692 return false; 693 } 694 } 695 696 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in) 697 { 698 if (devx_is_whitelist_cmd(cmd_in)) { 699 struct mlx5_ib_dev *dev; 700 701 if (c->devx_uid) 702 return c->devx_uid; 703 704 dev = to_mdev(c->ibucontext.device); 705 if (dev->devx_whitelist_uid) 706 return dev->devx_whitelist_uid; 707 708 return -EOPNOTSUPP; 709 } 710 711 if (!c->devx_uid) 712 return -EINVAL; 713 714 return c->devx_uid; 715 } 716 static bool devx_is_general_cmd(void *in) 717 { 718 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); 719 720 if (opcode >= MLX5_CMD_OP_GENERAL_START && 721 opcode < MLX5_CMD_OP_GENERAL_END) 722 return true; 723 724 switch (opcode) { 725 case MLX5_CMD_OP_QUERY_HCA_CAP: 726 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: 727 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT: 728 case MLX5_CMD_OP_QUERY_VPORT_STATE: 729 case MLX5_CMD_OP_QUERY_ADAPTER: 730 case MLX5_CMD_OP_QUERY_ISSI: 731 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT: 732 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS: 733 case MLX5_CMD_OP_QUERY_VNIC_ENV: 734 case MLX5_CMD_OP_QUERY_VPORT_COUNTER: 735 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG: 736 case MLX5_CMD_OP_NOP: 737 case MLX5_CMD_OP_QUERY_CONG_STATUS: 738 case MLX5_CMD_OP_QUERY_CONG_PARAMS: 739 case MLX5_CMD_OP_QUERY_CONG_STATISTICS: 740 return true; 741 default: 742 return false; 743 } 744 } 745 746 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)( 747 struct uverbs_attr_bundle *attrs) 748 { 749 struct mlx5_ib_ucontext *c; 750 struct mlx5_ib_dev *dev; 751 int user_vector; 752 int dev_eqn; 753 unsigned int irqn; 754 int err; 755 756 if (uverbs_copy_from(&user_vector, attrs, 757 MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC)) 758 return -EFAULT; 759 760 c = devx_ufile2uctx(attrs); 761 if (IS_ERR(c)) 762 return PTR_ERR(c); 763 dev = to_mdev(c->ibucontext.device); 764 765 err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn); 766 if (err < 0) 767 return err; 768 769 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN, 770 &dev_eqn, sizeof(dev_eqn))) 771 return -EFAULT; 772 773 return 0; 774 } 775 776 /* 777 *Security note: 778 * The hardware protection mechanism works like this: Each device object that 779 * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in 780 * the device specification manual) upon its creation. Then upon doorbell, 781 * hardware fetches the object context for which the doorbell was rang, and 782 * validates that the UAR through which the DB was rang matches the UAR ID 783 * of the object. 784 * If no match the doorbell is silently ignored by the hardware. Of course, 785 * the user cannot ring a doorbell on a UAR that was not mapped to it. 786 * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command 787 * mailboxes (except tagging them with UID), we expose to the user its UAR 788 * ID, so it can embed it in these objects in the expected specification 789 * format. So the only thing the user can do is hurt itself by creating a 790 * QP/SQ/CQ with a UAR ID other than his, and then in this case other users 791 * may ring a doorbell on its objects. 792 * The consequence of that will be that another user can schedule a QP/SQ 793 * of the buggy user for execution (just insert it to the hardware schedule 794 * queue or arm its CQ for event generation), no further harm is expected. 795 */ 796 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)( 797 struct uverbs_attr_bundle *attrs) 798 { 799 struct mlx5_ib_ucontext *c; 800 struct mlx5_ib_dev *dev; 801 u32 user_idx; 802 s32 dev_idx; 803 804 c = devx_ufile2uctx(attrs); 805 if (IS_ERR(c)) 806 return PTR_ERR(c); 807 dev = to_mdev(c->ibucontext.device); 808 809 if (uverbs_copy_from(&user_idx, attrs, 810 MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX)) 811 return -EFAULT; 812 813 dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true); 814 if (dev_idx < 0) 815 return dev_idx; 816 817 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX, 818 &dev_idx, sizeof(dev_idx))) 819 return -EFAULT; 820 821 return 0; 822 } 823 824 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)( 825 struct uverbs_attr_bundle *attrs) 826 { 827 struct mlx5_ib_ucontext *c; 828 struct mlx5_ib_dev *dev; 829 void *cmd_in = uverbs_attr_get_alloced_ptr( 830 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN); 831 int cmd_out_len = uverbs_attr_get_len(attrs, 832 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT); 833 void *cmd_out; 834 int err; 835 int uid; 836 837 c = devx_ufile2uctx(attrs); 838 if (IS_ERR(c)) 839 return PTR_ERR(c); 840 dev = to_mdev(c->ibucontext.device); 841 842 uid = devx_get_uid(c, cmd_in); 843 if (uid < 0) 844 return uid; 845 846 /* Only white list of some general HCA commands are allowed for this method. */ 847 if (!devx_is_general_cmd(cmd_in)) 848 return -EINVAL; 849 850 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 851 if (IS_ERR(cmd_out)) 852 return PTR_ERR(cmd_out); 853 854 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 855 err = mlx5_cmd_exec(dev->mdev, cmd_in, 856 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN), 857 cmd_out, cmd_out_len); 858 if (err) 859 return err; 860 861 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out, 862 cmd_out_len); 863 } 864 865 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din, 866 u32 *dinlen, 867 u32 *obj_id) 868 { 869 u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type); 870 u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid); 871 872 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 873 *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr); 874 875 MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id); 876 MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid); 877 878 switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) { 879 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: 880 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 881 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type); 882 break; 883 884 case MLX5_CMD_OP_CREATE_UMEM: 885 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 886 MLX5_CMD_OP_DESTROY_UMEM); 887 break; 888 case MLX5_CMD_OP_CREATE_MKEY: 889 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY); 890 break; 891 case MLX5_CMD_OP_CREATE_CQ: 892 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ); 893 break; 894 case MLX5_CMD_OP_ALLOC_PD: 895 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD); 896 break; 897 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN: 898 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 899 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN); 900 break; 901 case MLX5_CMD_OP_CREATE_RMP: 902 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP); 903 break; 904 case MLX5_CMD_OP_CREATE_SQ: 905 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ); 906 break; 907 case MLX5_CMD_OP_CREATE_RQ: 908 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ); 909 break; 910 case MLX5_CMD_OP_CREATE_RQT: 911 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT); 912 break; 913 case MLX5_CMD_OP_CREATE_TIR: 914 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR); 915 break; 916 case MLX5_CMD_OP_CREATE_TIS: 917 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS); 918 break; 919 case MLX5_CMD_OP_ALLOC_Q_COUNTER: 920 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 921 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 922 break; 923 case MLX5_CMD_OP_CREATE_FLOW_TABLE: 924 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in); 925 *obj_id = MLX5_GET(create_flow_table_out, out, table_id); 926 MLX5_SET(destroy_flow_table_in, din, other_vport, 927 MLX5_GET(create_flow_table_in, in, other_vport)); 928 MLX5_SET(destroy_flow_table_in, din, vport_number, 929 MLX5_GET(create_flow_table_in, in, vport_number)); 930 MLX5_SET(destroy_flow_table_in, din, table_type, 931 MLX5_GET(create_flow_table_in, in, table_type)); 932 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id); 933 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 934 MLX5_CMD_OP_DESTROY_FLOW_TABLE); 935 break; 936 case MLX5_CMD_OP_CREATE_FLOW_GROUP: 937 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in); 938 *obj_id = MLX5_GET(create_flow_group_out, out, group_id); 939 MLX5_SET(destroy_flow_group_in, din, other_vport, 940 MLX5_GET(create_flow_group_in, in, other_vport)); 941 MLX5_SET(destroy_flow_group_in, din, vport_number, 942 MLX5_GET(create_flow_group_in, in, vport_number)); 943 MLX5_SET(destroy_flow_group_in, din, table_type, 944 MLX5_GET(create_flow_group_in, in, table_type)); 945 MLX5_SET(destroy_flow_group_in, din, table_id, 946 MLX5_GET(create_flow_group_in, in, table_id)); 947 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id); 948 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 949 MLX5_CMD_OP_DESTROY_FLOW_GROUP); 950 break; 951 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: 952 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in); 953 *obj_id = MLX5_GET(set_fte_in, in, flow_index); 954 MLX5_SET(delete_fte_in, din, other_vport, 955 MLX5_GET(set_fte_in, in, other_vport)); 956 MLX5_SET(delete_fte_in, din, vport_number, 957 MLX5_GET(set_fte_in, in, vport_number)); 958 MLX5_SET(delete_fte_in, din, table_type, 959 MLX5_GET(set_fte_in, in, table_type)); 960 MLX5_SET(delete_fte_in, din, table_id, 961 MLX5_GET(set_fte_in, in, table_id)); 962 MLX5_SET(delete_fte_in, din, flow_index, *obj_id); 963 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 964 MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY); 965 break; 966 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: 967 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 968 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER); 969 break; 970 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT: 971 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 972 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT); 973 break; 974 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT: 975 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 976 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT); 977 break; 978 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT: 979 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in); 980 *obj_id = MLX5_GET(create_scheduling_element_out, out, 981 scheduling_element_id); 982 MLX5_SET(destroy_scheduling_element_in, din, 983 scheduling_hierarchy, 984 MLX5_GET(create_scheduling_element_in, in, 985 scheduling_hierarchy)); 986 MLX5_SET(destroy_scheduling_element_in, din, 987 scheduling_element_id, *obj_id); 988 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 989 MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT); 990 break; 991 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT: 992 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in); 993 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port); 994 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id); 995 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 996 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT); 997 break; 998 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY: 999 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in); 1000 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index); 1001 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id); 1002 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 1003 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY); 1004 break; 1005 case MLX5_CMD_OP_CREATE_QP: 1006 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP); 1007 break; 1008 case MLX5_CMD_OP_CREATE_SRQ: 1009 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ); 1010 break; 1011 case MLX5_CMD_OP_CREATE_XRC_SRQ: 1012 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, 1013 MLX5_CMD_OP_DESTROY_XRC_SRQ); 1014 break; 1015 case MLX5_CMD_OP_CREATE_DCT: 1016 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT); 1017 break; 1018 case MLX5_CMD_OP_CREATE_XRQ: 1019 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ); 1020 break; 1021 case MLX5_CMD_OP_ATTACH_TO_MCG: 1022 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in); 1023 MLX5_SET(detach_from_mcg_in, din, qpn, 1024 MLX5_GET(attach_to_mcg_in, in, qpn)); 1025 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid), 1026 MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid), 1027 MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid)); 1028 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG); 1029 break; 1030 case MLX5_CMD_OP_ALLOC_XRCD: 1031 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD); 1032 break; 1033 default: 1034 /* The entry must match to one of the devx_is_obj_create_cmd */ 1035 WARN_ON(true); 1036 break; 1037 } 1038 } 1039 1040 static int devx_handle_mkey_indirect(struct devx_obj *obj, 1041 struct mlx5_ib_dev *dev, 1042 void *in, void *out) 1043 { 1044 struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table; 1045 struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr; 1046 unsigned long flags; 1047 struct mlx5_core_mkey *mkey; 1048 void *mkc; 1049 u8 key; 1050 int err; 1051 1052 mkey = &devx_mr->mmkey; 1053 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1054 key = MLX5_GET(mkc, mkc, mkey_7_0); 1055 mkey->key = mlx5_idx_to_mkey( 1056 MLX5_GET(create_mkey_out, out, mkey_index)) | key; 1057 mkey->type = MLX5_MKEY_INDIRECT_DEVX; 1058 mkey->iova = MLX5_GET64(mkc, mkc, start_addr); 1059 mkey->size = MLX5_GET64(mkc, mkc, len); 1060 mkey->pd = MLX5_GET(mkc, mkc, pd); 1061 devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size); 1062 1063 write_lock_irqsave(&table->lock, flags); 1064 err = radix_tree_insert(&table->tree, mlx5_base_mkey(mkey->key), 1065 mkey); 1066 write_unlock_irqrestore(&table->lock, flags); 1067 return err; 1068 } 1069 1070 static int devx_handle_mkey_create(struct mlx5_ib_dev *dev, 1071 struct devx_obj *obj, 1072 void *in, int in_len) 1073 { 1074 int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) + 1075 MLX5_FLD_SZ_BYTES(create_mkey_in, 1076 memory_key_mkey_entry); 1077 void *mkc; 1078 u8 access_mode; 1079 1080 if (in_len < min_len) 1081 return -EINVAL; 1082 1083 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1084 1085 access_mode = MLX5_GET(mkc, mkc, access_mode_1_0); 1086 access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2; 1087 1088 if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS || 1089 access_mode == MLX5_MKC_ACCESS_MODE_KSM) { 1090 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 1091 obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY; 1092 return 0; 1093 } 1094 1095 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1); 1096 return 0; 1097 } 1098 1099 static void devx_free_indirect_mkey(struct rcu_head *rcu) 1100 { 1101 kfree(container_of(rcu, struct devx_obj, devx_mr.rcu)); 1102 } 1103 1104 /* This function to delete from the radix tree needs to be called before 1105 * destroying the underlying mkey. Otherwise a race might occur in case that 1106 * other thread will get the same mkey before this one will be deleted, 1107 * in that case it will fail via inserting to the tree its own data. 1108 * 1109 * Note: 1110 * An error in the destroy is not expected unless there is some other indirect 1111 * mkey which points to this one. In a kernel cleanup flow it will be just 1112 * destroyed in the iterative destruction call. In a user flow, in case 1113 * the application didn't close in the expected order it's its own problem, 1114 * the mkey won't be part of the tree, in both cases the kernel is safe. 1115 */ 1116 static void devx_cleanup_mkey(struct devx_obj *obj) 1117 { 1118 struct mlx5_mkey_table *table = &obj->mdev->priv.mkey_table; 1119 unsigned long flags; 1120 1121 write_lock_irqsave(&table->lock, flags); 1122 radix_tree_delete(&table->tree, mlx5_base_mkey(obj->devx_mr.mmkey.key)); 1123 write_unlock_irqrestore(&table->lock, flags); 1124 } 1125 1126 static int devx_obj_cleanup(struct ib_uobject *uobject, 1127 enum rdma_remove_reason why, 1128 struct uverbs_attr_bundle *attrs) 1129 { 1130 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 1131 struct devx_obj *obj = uobject->object; 1132 int ret; 1133 1134 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) 1135 devx_cleanup_mkey(obj); 1136 1137 if (obj->flags & DEVX_OBJ_FLAGS_DCT) 1138 ret = mlx5_core_destroy_dct(obj->mdev, &obj->core_dct); 1139 else 1140 ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, 1141 sizeof(out)); 1142 if (ib_is_destroy_retryable(ret, why, uobject)) 1143 return ret; 1144 1145 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) { 1146 struct mlx5_ib_dev *dev = 1147 mlx5_udata_to_mdev(&attrs->driver_udata); 1148 1149 call_srcu(&dev->mr_srcu, &obj->devx_mr.rcu, 1150 devx_free_indirect_mkey); 1151 return ret; 1152 } 1153 1154 kfree(obj); 1155 return ret; 1156 } 1157 1158 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( 1159 struct uverbs_attr_bundle *attrs) 1160 { 1161 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN); 1162 int cmd_out_len = uverbs_attr_get_len(attrs, 1163 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT); 1164 int cmd_in_len = uverbs_attr_get_len(attrs, 1165 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN); 1166 void *cmd_out; 1167 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1168 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE); 1169 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1170 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1171 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); 1172 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 1173 struct devx_obj *obj; 1174 int err; 1175 int uid; 1176 u32 obj_id; 1177 u16 opcode; 1178 1179 uid = devx_get_uid(c, cmd_in); 1180 if (uid < 0) 1181 return uid; 1182 1183 if (!devx_is_obj_create_cmd(cmd_in, &opcode)) 1184 return -EINVAL; 1185 1186 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 1187 if (IS_ERR(cmd_out)) 1188 return PTR_ERR(cmd_out); 1189 1190 obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL); 1191 if (!obj) 1192 return -ENOMEM; 1193 1194 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1195 if (opcode == MLX5_CMD_OP_CREATE_MKEY) { 1196 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len); 1197 if (err) 1198 goto obj_free; 1199 } else { 1200 devx_set_umem_valid(cmd_in); 1201 } 1202 1203 if (opcode == MLX5_CMD_OP_CREATE_DCT) { 1204 obj->flags |= DEVX_OBJ_FLAGS_DCT; 1205 err = mlx5_core_create_dct(dev->mdev, &obj->core_dct, 1206 cmd_in, cmd_in_len, 1207 cmd_out, cmd_out_len); 1208 } else { 1209 err = mlx5_cmd_exec(dev->mdev, cmd_in, 1210 cmd_in_len, 1211 cmd_out, cmd_out_len); 1212 } 1213 1214 if (err) 1215 goto obj_free; 1216 1217 uobj->object = obj; 1218 obj->mdev = dev->mdev; 1219 devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen, 1220 &obj_id); 1221 WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32)); 1222 1223 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) { 1224 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out); 1225 if (err) 1226 goto obj_destroy; 1227 } 1228 1229 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len); 1230 if (err) 1231 goto err_copy; 1232 1233 obj->obj_id = get_enc_obj_id(opcode, obj_id); 1234 return 0; 1235 1236 err_copy: 1237 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) 1238 devx_cleanup_mkey(obj); 1239 obj_destroy: 1240 if (obj->flags & DEVX_OBJ_FLAGS_DCT) 1241 mlx5_core_destroy_dct(obj->mdev, &obj->core_dct); 1242 else 1243 mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, 1244 sizeof(out)); 1245 obj_free: 1246 kfree(obj); 1247 return err; 1248 } 1249 1250 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)( 1251 struct uverbs_attr_bundle *attrs) 1252 { 1253 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN); 1254 int cmd_out_len = uverbs_attr_get_len(attrs, 1255 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT); 1256 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 1257 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE); 1258 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1259 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1260 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); 1261 void *cmd_out; 1262 int err; 1263 int uid; 1264 1265 uid = devx_get_uid(c, cmd_in); 1266 if (uid < 0) 1267 return uid; 1268 1269 if (!devx_is_obj_modify_cmd(cmd_in)) 1270 return -EINVAL; 1271 1272 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in)) 1273 return -EINVAL; 1274 1275 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 1276 if (IS_ERR(cmd_out)) 1277 return PTR_ERR(cmd_out); 1278 1279 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1280 devx_set_umem_valid(cmd_in); 1281 1282 err = mlx5_cmd_exec(mdev->mdev, cmd_in, 1283 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN), 1284 cmd_out, cmd_out_len); 1285 if (err) 1286 return err; 1287 1288 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT, 1289 cmd_out, cmd_out_len); 1290 } 1291 1292 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)( 1293 struct uverbs_attr_bundle *attrs) 1294 { 1295 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN); 1296 int cmd_out_len = uverbs_attr_get_len(attrs, 1297 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT); 1298 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 1299 MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE); 1300 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1301 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1302 void *cmd_out; 1303 int err; 1304 int uid; 1305 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); 1306 1307 uid = devx_get_uid(c, cmd_in); 1308 if (uid < 0) 1309 return uid; 1310 1311 if (!devx_is_obj_query_cmd(cmd_in)) 1312 return -EINVAL; 1313 1314 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in)) 1315 return -EINVAL; 1316 1317 cmd_out = uverbs_zalloc(attrs, cmd_out_len); 1318 if (IS_ERR(cmd_out)) 1319 return PTR_ERR(cmd_out); 1320 1321 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1322 err = mlx5_cmd_exec(mdev->mdev, cmd_in, 1323 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN), 1324 cmd_out, cmd_out_len); 1325 if (err) 1326 return err; 1327 1328 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT, 1329 cmd_out, cmd_out_len); 1330 } 1331 1332 struct devx_async_event_queue { 1333 spinlock_t lock; 1334 wait_queue_head_t poll_wait; 1335 struct list_head event_list; 1336 atomic_t bytes_in_use; 1337 u8 is_destroyed:1; 1338 }; 1339 1340 struct devx_async_cmd_event_file { 1341 struct ib_uobject uobj; 1342 struct devx_async_event_queue ev_queue; 1343 struct mlx5_async_ctx async_ctx; 1344 }; 1345 1346 static void devx_init_event_queue(struct devx_async_event_queue *ev_queue) 1347 { 1348 spin_lock_init(&ev_queue->lock); 1349 INIT_LIST_HEAD(&ev_queue->event_list); 1350 init_waitqueue_head(&ev_queue->poll_wait); 1351 atomic_set(&ev_queue->bytes_in_use, 0); 1352 ev_queue->is_destroyed = 0; 1353 } 1354 1355 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)( 1356 struct uverbs_attr_bundle *attrs) 1357 { 1358 struct devx_async_cmd_event_file *ev_file; 1359 1360 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1361 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE); 1362 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 1363 1364 ev_file = container_of(uobj, struct devx_async_cmd_event_file, 1365 uobj); 1366 devx_init_event_queue(&ev_file->ev_queue); 1367 mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx); 1368 return 0; 1369 } 1370 1371 static void devx_query_callback(int status, struct mlx5_async_work *context) 1372 { 1373 struct devx_async_data *async_data = 1374 container_of(context, struct devx_async_data, cb_work); 1375 struct ib_uobject *fd_uobj = async_data->fd_uobj; 1376 struct devx_async_cmd_event_file *ev_file; 1377 struct devx_async_event_queue *ev_queue; 1378 unsigned long flags; 1379 1380 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file, 1381 uobj); 1382 ev_queue = &ev_file->ev_queue; 1383 1384 spin_lock_irqsave(&ev_queue->lock, flags); 1385 list_add_tail(&async_data->list, &ev_queue->event_list); 1386 spin_unlock_irqrestore(&ev_queue->lock, flags); 1387 1388 wake_up_interruptible(&ev_queue->poll_wait); 1389 fput(fd_uobj->object); 1390 } 1391 1392 #define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */ 1393 1394 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)( 1395 struct uverbs_attr_bundle *attrs) 1396 { 1397 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, 1398 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN); 1399 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1400 attrs, 1401 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE); 1402 u16 cmd_out_len; 1403 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1404 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1405 struct ib_uobject *fd_uobj; 1406 int err; 1407 int uid; 1408 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); 1409 struct devx_async_cmd_event_file *ev_file; 1410 struct devx_async_data *async_data; 1411 1412 uid = devx_get_uid(c, cmd_in); 1413 if (uid < 0) 1414 return uid; 1415 1416 if (!devx_is_obj_query_cmd(cmd_in)) 1417 return -EINVAL; 1418 1419 err = uverbs_get_const(&cmd_out_len, attrs, 1420 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN); 1421 if (err) 1422 return err; 1423 1424 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in)) 1425 return -EINVAL; 1426 1427 fd_uobj = uverbs_attr_get_uobject(attrs, 1428 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD); 1429 if (IS_ERR(fd_uobj)) 1430 return PTR_ERR(fd_uobj); 1431 1432 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file, 1433 uobj); 1434 1435 if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) > 1436 MAX_ASYNC_BYTES_IN_USE) { 1437 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use); 1438 return -EAGAIN; 1439 } 1440 1441 async_data = kvzalloc(struct_size(async_data, hdr.out_data, 1442 cmd_out_len), GFP_KERNEL); 1443 if (!async_data) { 1444 err = -ENOMEM; 1445 goto sub_bytes; 1446 } 1447 1448 err = uverbs_copy_from(&async_data->hdr.wr_id, attrs, 1449 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID); 1450 if (err) 1451 goto free_async; 1452 1453 async_data->cmd_out_len = cmd_out_len; 1454 async_data->mdev = mdev; 1455 async_data->fd_uobj = fd_uobj; 1456 1457 get_file(fd_uobj->object); 1458 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid); 1459 err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in, 1460 uverbs_attr_get_len(attrs, 1461 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN), 1462 async_data->hdr.out_data, 1463 async_data->cmd_out_len, 1464 devx_query_callback, &async_data->cb_work); 1465 1466 if (err) 1467 goto cb_err; 1468 1469 return 0; 1470 1471 cb_err: 1472 fput(fd_uobj->object); 1473 free_async: 1474 kvfree(async_data); 1475 sub_bytes: 1476 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use); 1477 return err; 1478 } 1479 1480 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, 1481 struct uverbs_attr_bundle *attrs, 1482 struct devx_umem *obj) 1483 { 1484 u64 addr; 1485 size_t size; 1486 u32 access; 1487 int npages; 1488 int err; 1489 u32 page_mask; 1490 1491 if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) || 1492 uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN)) 1493 return -EFAULT; 1494 1495 err = uverbs_get_flags32(&access, attrs, 1496 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, 1497 IB_ACCESS_LOCAL_WRITE | 1498 IB_ACCESS_REMOTE_WRITE | 1499 IB_ACCESS_REMOTE_READ); 1500 if (err) 1501 return err; 1502 1503 err = ib_check_mr_access(access); 1504 if (err) 1505 return err; 1506 1507 obj->umem = ib_umem_get(&attrs->driver_udata, addr, size, access, 0); 1508 if (IS_ERR(obj->umem)) 1509 return PTR_ERR(obj->umem); 1510 1511 mlx5_ib_cont_pages(obj->umem, obj->umem->address, 1512 MLX5_MKEY_PAGE_SHIFT_MASK, &npages, 1513 &obj->page_shift, &obj->ncont, NULL); 1514 1515 if (!npages) { 1516 ib_umem_release(obj->umem); 1517 return -EINVAL; 1518 } 1519 1520 page_mask = (1 << obj->page_shift) - 1; 1521 obj->page_offset = obj->umem->address & page_mask; 1522 1523 return 0; 1524 } 1525 1526 static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs, 1527 struct devx_umem *obj, 1528 struct devx_umem_reg_cmd *cmd) 1529 { 1530 cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) + 1531 (MLX5_ST_SZ_BYTES(mtt) * obj->ncont); 1532 cmd->in = uverbs_zalloc(attrs, cmd->inlen); 1533 return PTR_ERR_OR_ZERO(cmd->in); 1534 } 1535 1536 static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev, 1537 struct devx_umem *obj, 1538 struct devx_umem_reg_cmd *cmd) 1539 { 1540 void *umem; 1541 __be64 *mtt; 1542 1543 umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem); 1544 mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt); 1545 1546 MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM); 1547 MLX5_SET64(umem, umem, num_of_mtt, obj->ncont); 1548 MLX5_SET(umem, umem, log_page_size, obj->page_shift - 1549 MLX5_ADAPTER_PAGE_SHIFT); 1550 MLX5_SET(umem, umem, page_offset, obj->page_offset); 1551 mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt, 1552 (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) | 1553 MLX5_IB_MTT_READ); 1554 } 1555 1556 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)( 1557 struct uverbs_attr_bundle *attrs) 1558 { 1559 struct devx_umem_reg_cmd cmd; 1560 struct devx_umem *obj; 1561 struct ib_uobject *uobj = uverbs_attr_get_uobject( 1562 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE); 1563 u32 obj_id; 1564 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( 1565 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 1566 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); 1567 int err; 1568 1569 if (!c->devx_uid) 1570 return -EINVAL; 1571 1572 obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL); 1573 if (!obj) 1574 return -ENOMEM; 1575 1576 err = devx_umem_get(dev, &c->ibucontext, attrs, obj); 1577 if (err) 1578 goto err_obj_free; 1579 1580 err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd); 1581 if (err) 1582 goto err_umem_release; 1583 1584 devx_umem_reg_cmd_build(dev, obj, &cmd); 1585 1586 MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid); 1587 err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out, 1588 sizeof(cmd.out)); 1589 if (err) 1590 goto err_umem_release; 1591 1592 obj->mdev = dev->mdev; 1593 uobj->object = obj; 1594 devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id); 1595 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id)); 1596 if (err) 1597 goto err_umem_destroy; 1598 1599 return 0; 1600 1601 err_umem_destroy: 1602 mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out)); 1603 err_umem_release: 1604 ib_umem_release(obj->umem); 1605 err_obj_free: 1606 kfree(obj); 1607 return err; 1608 } 1609 1610 static int devx_umem_cleanup(struct ib_uobject *uobject, 1611 enum rdma_remove_reason why, 1612 struct uverbs_attr_bundle *attrs) 1613 { 1614 struct devx_umem *obj = uobject->object; 1615 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 1616 int err; 1617 1618 err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out)); 1619 if (ib_is_destroy_retryable(err, why, uobject)) 1620 return err; 1621 1622 ib_umem_release(obj->umem); 1623 kfree(obj); 1624 return 0; 1625 } 1626 1627 static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf, 1628 size_t count, loff_t *pos) 1629 { 1630 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data; 1631 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; 1632 struct devx_async_data *event; 1633 int ret = 0; 1634 size_t eventsz; 1635 1636 spin_lock_irq(&ev_queue->lock); 1637 1638 while (list_empty(&ev_queue->event_list)) { 1639 spin_unlock_irq(&ev_queue->lock); 1640 1641 if (filp->f_flags & O_NONBLOCK) 1642 return -EAGAIN; 1643 1644 if (wait_event_interruptible( 1645 ev_queue->poll_wait, 1646 (!list_empty(&ev_queue->event_list) || 1647 ev_queue->is_destroyed))) { 1648 return -ERESTARTSYS; 1649 } 1650 1651 if (list_empty(&ev_queue->event_list) && 1652 ev_queue->is_destroyed) 1653 return -EIO; 1654 1655 spin_lock_irq(&ev_queue->lock); 1656 } 1657 1658 event = list_entry(ev_queue->event_list.next, 1659 struct devx_async_data, list); 1660 eventsz = event->cmd_out_len + 1661 sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr); 1662 1663 if (eventsz > count) { 1664 spin_unlock_irq(&ev_queue->lock); 1665 return -ENOSPC; 1666 } 1667 1668 list_del(ev_queue->event_list.next); 1669 spin_unlock_irq(&ev_queue->lock); 1670 1671 if (copy_to_user(buf, &event->hdr, eventsz)) 1672 ret = -EFAULT; 1673 else 1674 ret = eventsz; 1675 1676 atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use); 1677 kvfree(event); 1678 return ret; 1679 } 1680 1681 static int devx_async_cmd_event_close(struct inode *inode, struct file *filp) 1682 { 1683 struct ib_uobject *uobj = filp->private_data; 1684 struct devx_async_cmd_event_file *comp_ev_file = container_of( 1685 uobj, struct devx_async_cmd_event_file, uobj); 1686 struct devx_async_data *entry, *tmp; 1687 1688 spin_lock_irq(&comp_ev_file->ev_queue.lock); 1689 list_for_each_entry_safe(entry, tmp, 1690 &comp_ev_file->ev_queue.event_list, list) 1691 kvfree(entry); 1692 spin_unlock_irq(&comp_ev_file->ev_queue.lock); 1693 1694 uverbs_close_fd(filp); 1695 return 0; 1696 } 1697 1698 static __poll_t devx_async_cmd_event_poll(struct file *filp, 1699 struct poll_table_struct *wait) 1700 { 1701 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data; 1702 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; 1703 __poll_t pollflags = 0; 1704 1705 poll_wait(filp, &ev_queue->poll_wait, wait); 1706 1707 spin_lock_irq(&ev_queue->lock); 1708 if (ev_queue->is_destroyed) 1709 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 1710 else if (!list_empty(&ev_queue->event_list)) 1711 pollflags = EPOLLIN | EPOLLRDNORM; 1712 spin_unlock_irq(&ev_queue->lock); 1713 1714 return pollflags; 1715 } 1716 1717 static const struct file_operations devx_async_cmd_event_fops = { 1718 .owner = THIS_MODULE, 1719 .read = devx_async_cmd_event_read, 1720 .poll = devx_async_cmd_event_poll, 1721 .release = devx_async_cmd_event_close, 1722 .llseek = no_llseek, 1723 }; 1724 1725 static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj, 1726 enum rdma_remove_reason why) 1727 { 1728 struct devx_async_cmd_event_file *comp_ev_file = 1729 container_of(uobj, struct devx_async_cmd_event_file, 1730 uobj); 1731 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue; 1732 1733 spin_lock_irq(&ev_queue->lock); 1734 ev_queue->is_destroyed = 1; 1735 spin_unlock_irq(&ev_queue->lock); 1736 1737 if (why == RDMA_REMOVE_DRIVER_REMOVE) 1738 wake_up_interruptible(&ev_queue->poll_wait); 1739 1740 mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx); 1741 return 0; 1742 }; 1743 1744 DECLARE_UVERBS_NAMED_METHOD( 1745 MLX5_IB_METHOD_DEVX_UMEM_REG, 1746 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE, 1747 MLX5_IB_OBJECT_DEVX_UMEM, 1748 UVERBS_ACCESS_NEW, 1749 UA_MANDATORY), 1750 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR, 1751 UVERBS_ATTR_TYPE(u64), 1752 UA_MANDATORY), 1753 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN, 1754 UVERBS_ATTR_TYPE(u64), 1755 UA_MANDATORY), 1756 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, 1757 enum ib_access_flags), 1758 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, 1759 UVERBS_ATTR_TYPE(u32), 1760 UA_MANDATORY)); 1761 1762 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 1763 MLX5_IB_METHOD_DEVX_UMEM_DEREG, 1764 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE, 1765 MLX5_IB_OBJECT_DEVX_UMEM, 1766 UVERBS_ACCESS_DESTROY, 1767 UA_MANDATORY)); 1768 1769 DECLARE_UVERBS_NAMED_METHOD( 1770 MLX5_IB_METHOD_DEVX_QUERY_EQN, 1771 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC, 1772 UVERBS_ATTR_TYPE(u32), 1773 UA_MANDATORY), 1774 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN, 1775 UVERBS_ATTR_TYPE(u32), 1776 UA_MANDATORY)); 1777 1778 DECLARE_UVERBS_NAMED_METHOD( 1779 MLX5_IB_METHOD_DEVX_QUERY_UAR, 1780 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX, 1781 UVERBS_ATTR_TYPE(u32), 1782 UA_MANDATORY), 1783 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX, 1784 UVERBS_ATTR_TYPE(u32), 1785 UA_MANDATORY)); 1786 1787 DECLARE_UVERBS_NAMED_METHOD( 1788 MLX5_IB_METHOD_DEVX_OTHER, 1789 UVERBS_ATTR_PTR_IN( 1790 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN, 1791 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1792 UA_MANDATORY, 1793 UA_ALLOC_AND_COPY), 1794 UVERBS_ATTR_PTR_OUT( 1795 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, 1796 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1797 UA_MANDATORY)); 1798 1799 DECLARE_UVERBS_NAMED_METHOD( 1800 MLX5_IB_METHOD_DEVX_OBJ_CREATE, 1801 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE, 1802 MLX5_IB_OBJECT_DEVX_OBJ, 1803 UVERBS_ACCESS_NEW, 1804 UA_MANDATORY), 1805 UVERBS_ATTR_PTR_IN( 1806 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN, 1807 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1808 UA_MANDATORY, 1809 UA_ALLOC_AND_COPY), 1810 UVERBS_ATTR_PTR_OUT( 1811 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, 1812 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1813 UA_MANDATORY)); 1814 1815 DECLARE_UVERBS_NAMED_METHOD_DESTROY( 1816 MLX5_IB_METHOD_DEVX_OBJ_DESTROY, 1817 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE, 1818 MLX5_IB_OBJECT_DEVX_OBJ, 1819 UVERBS_ACCESS_DESTROY, 1820 UA_MANDATORY)); 1821 1822 DECLARE_UVERBS_NAMED_METHOD( 1823 MLX5_IB_METHOD_DEVX_OBJ_MODIFY, 1824 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE, 1825 UVERBS_IDR_ANY_OBJECT, 1826 UVERBS_ACCESS_WRITE, 1827 UA_MANDATORY), 1828 UVERBS_ATTR_PTR_IN( 1829 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN, 1830 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1831 UA_MANDATORY, 1832 UA_ALLOC_AND_COPY), 1833 UVERBS_ATTR_PTR_OUT( 1834 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT, 1835 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1836 UA_MANDATORY)); 1837 1838 DECLARE_UVERBS_NAMED_METHOD( 1839 MLX5_IB_METHOD_DEVX_OBJ_QUERY, 1840 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE, 1841 UVERBS_IDR_ANY_OBJECT, 1842 UVERBS_ACCESS_READ, 1843 UA_MANDATORY), 1844 UVERBS_ATTR_PTR_IN( 1845 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN, 1846 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1847 UA_MANDATORY, 1848 UA_ALLOC_AND_COPY), 1849 UVERBS_ATTR_PTR_OUT( 1850 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT, 1851 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)), 1852 UA_MANDATORY)); 1853 1854 DECLARE_UVERBS_NAMED_METHOD( 1855 MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY, 1856 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE, 1857 UVERBS_IDR_ANY_OBJECT, 1858 UVERBS_ACCESS_READ, 1859 UA_MANDATORY), 1860 UVERBS_ATTR_PTR_IN( 1861 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN, 1862 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)), 1863 UA_MANDATORY, 1864 UA_ALLOC_AND_COPY), 1865 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN, 1866 u16, UA_MANDATORY), 1867 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD, 1868 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1869 UVERBS_ACCESS_READ, 1870 UA_MANDATORY), 1871 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID, 1872 UVERBS_ATTR_TYPE(u64), 1873 UA_MANDATORY)); 1874 1875 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX, 1876 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER), 1877 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR), 1878 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN)); 1879 1880 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ, 1881 UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup), 1882 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE), 1883 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY), 1884 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY), 1885 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY), 1886 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)); 1887 1888 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM, 1889 UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup), 1890 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG), 1891 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG)); 1892 1893 1894 DECLARE_UVERBS_NAMED_METHOD( 1895 MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC, 1896 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE, 1897 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1898 UVERBS_ACCESS_NEW, 1899 UA_MANDATORY)); 1900 1901 DECLARE_UVERBS_NAMED_OBJECT( 1902 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1903 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file), 1904 devx_hot_unplug_async_cmd_event_file, 1905 &devx_async_cmd_event_fops, "[devx_async_cmd]", 1906 O_RDONLY), 1907 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)); 1908 1909 static bool devx_is_supported(struct ib_device *device) 1910 { 1911 struct mlx5_ib_dev *dev = to_mdev(device); 1912 1913 return MLX5_CAP_GEN(dev->mdev, log_max_uctx); 1914 } 1915 1916 const struct uapi_definition mlx5_ib_devx_defs[] = { 1917 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1918 MLX5_IB_OBJECT_DEVX, 1919 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1920 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1921 MLX5_IB_OBJECT_DEVX_OBJ, 1922 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1923 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1924 MLX5_IB_OBJECT_DEVX_UMEM, 1925 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1926 UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 1927 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, 1928 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), 1929 {}, 1930 }; 1931