1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2011-2014, Intel Corporation. 4 * Copyright (c) 2017-2021 Christoph Hellwig. 5 */ 6 #include <linux/ptrace.h> /* for force_successful_syscall_return */ 7 #include <linux/nvme_ioctl.h> 8 #include "nvme.h" 9 10 /* 11 * Convert integer values from ioctl structures to user pointers, silently 12 * ignoring the upper bits in the compat case to match behaviour of 32-bit 13 * kernels. 14 */ 15 static void __user *nvme_to_user_ptr(uintptr_t ptrval) 16 { 17 if (in_compat_syscall()) 18 ptrval = (compat_uptr_t)ptrval; 19 return (void __user *)ptrval; 20 } 21 22 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf, 23 unsigned len, u32 seed, bool write) 24 { 25 struct bio_integrity_payload *bip; 26 int ret = -ENOMEM; 27 void *buf; 28 29 buf = kmalloc(len, GFP_KERNEL); 30 if (!buf) 31 goto out; 32 33 ret = -EFAULT; 34 if (write && copy_from_user(buf, ubuf, len)) 35 goto out_free_meta; 36 37 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1); 38 if (IS_ERR(bip)) { 39 ret = PTR_ERR(bip); 40 goto out_free_meta; 41 } 42 43 bip->bip_iter.bi_size = len; 44 bip->bip_iter.bi_sector = seed; 45 ret = bio_integrity_add_page(bio, virt_to_page(buf), len, 46 offset_in_page(buf)); 47 if (ret == len) 48 return buf; 49 ret = -ENOMEM; 50 out_free_meta: 51 kfree(buf); 52 out: 53 return ERR_PTR(ret); 54 } 55 56 static int nvme_submit_user_cmd(struct request_queue *q, 57 struct nvme_command *cmd, void __user *ubuffer, 58 unsigned bufflen, void __user *meta_buffer, unsigned meta_len, 59 u32 meta_seed, u64 *result, unsigned timeout, bool vec) 60 { 61 bool write = nvme_is_write(cmd); 62 struct nvme_ns *ns = q->queuedata; 63 struct block_device *bdev = ns ? ns->disk->part0 : NULL; 64 struct request *req; 65 struct bio *bio = NULL; 66 void *meta = NULL; 67 int ret; 68 69 req = blk_mq_alloc_request(q, nvme_req_op(cmd), 0); 70 if (IS_ERR(req)) 71 return PTR_ERR(req); 72 nvme_init_request(req, cmd); 73 74 if (timeout) 75 req->timeout = timeout; 76 nvme_req(req)->flags |= NVME_REQ_USERCMD; 77 78 if (ubuffer && bufflen) { 79 if (!vec) 80 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen, 81 GFP_KERNEL); 82 else { 83 struct iovec fast_iov[UIO_FASTIOV]; 84 struct iovec *iov = fast_iov; 85 struct iov_iter iter; 86 87 ret = import_iovec(rq_data_dir(req), ubuffer, bufflen, 88 UIO_FASTIOV, &iov, &iter); 89 if (ret < 0) 90 goto out; 91 ret = blk_rq_map_user_iov(q, req, NULL, &iter, 92 GFP_KERNEL); 93 kfree(iov); 94 } 95 if (ret) 96 goto out; 97 bio = req->bio; 98 if (bdev) 99 bio_set_dev(bio, bdev); 100 if (bdev && meta_buffer && meta_len) { 101 meta = nvme_add_user_metadata(bio, meta_buffer, meta_len, 102 meta_seed, write); 103 if (IS_ERR(meta)) { 104 ret = PTR_ERR(meta); 105 goto out_unmap; 106 } 107 req->cmd_flags |= REQ_INTEGRITY; 108 } 109 } 110 111 ret = nvme_execute_passthru_rq(req); 112 if (result) 113 *result = le64_to_cpu(nvme_req(req)->result.u64); 114 if (meta && !ret && !write) { 115 if (copy_to_user(meta_buffer, meta, meta_len)) 116 ret = -EFAULT; 117 } 118 kfree(meta); 119 out_unmap: 120 if (bio) 121 blk_rq_unmap_user(bio); 122 out: 123 blk_mq_free_request(req); 124 return ret; 125 } 126 127 128 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 129 { 130 struct nvme_user_io io; 131 struct nvme_command c; 132 unsigned length, meta_len; 133 void __user *metadata; 134 135 if (copy_from_user(&io, uio, sizeof(io))) 136 return -EFAULT; 137 if (io.flags) 138 return -EINVAL; 139 140 switch (io.opcode) { 141 case nvme_cmd_write: 142 case nvme_cmd_read: 143 case nvme_cmd_compare: 144 break; 145 default: 146 return -EINVAL; 147 } 148 149 length = (io.nblocks + 1) << ns->lba_shift; 150 151 if ((io.control & NVME_RW_PRINFO_PRACT) && 152 ns->ms == sizeof(struct t10_pi_tuple)) { 153 /* 154 * Protection information is stripped/inserted by the 155 * controller. 156 */ 157 if (nvme_to_user_ptr(io.metadata)) 158 return -EINVAL; 159 meta_len = 0; 160 metadata = NULL; 161 } else { 162 meta_len = (io.nblocks + 1) * ns->ms; 163 metadata = nvme_to_user_ptr(io.metadata); 164 } 165 166 if (ns->features & NVME_NS_EXT_LBAS) { 167 length += meta_len; 168 meta_len = 0; 169 } else if (meta_len) { 170 if ((io.metadata & 3) || !io.metadata) 171 return -EINVAL; 172 } 173 174 memset(&c, 0, sizeof(c)); 175 c.rw.opcode = io.opcode; 176 c.rw.flags = io.flags; 177 c.rw.nsid = cpu_to_le32(ns->head->ns_id); 178 c.rw.slba = cpu_to_le64(io.slba); 179 c.rw.length = cpu_to_le16(io.nblocks); 180 c.rw.control = cpu_to_le16(io.control); 181 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt); 182 c.rw.reftag = cpu_to_le32(io.reftag); 183 c.rw.apptag = cpu_to_le16(io.apptag); 184 c.rw.appmask = cpu_to_le16(io.appmask); 185 186 return nvme_submit_user_cmd(ns->queue, &c, 187 nvme_to_user_ptr(io.addr), length, 188 metadata, meta_len, lower_32_bits(io.slba), NULL, 0, 189 false); 190 } 191 192 static bool nvme_validate_passthru_nsid(struct nvme_ctrl *ctrl, 193 struct nvme_ns *ns, __u32 nsid) 194 { 195 if (ns && nsid != ns->head->ns_id) { 196 dev_err(ctrl->device, 197 "%s: nsid (%u) in cmd does not match nsid (%u)" 198 "of namespace\n", 199 current->comm, nsid, ns->head->ns_id); 200 return false; 201 } 202 203 return true; 204 } 205 206 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 207 struct nvme_passthru_cmd __user *ucmd) 208 { 209 struct nvme_passthru_cmd cmd; 210 struct nvme_command c; 211 unsigned timeout = 0; 212 u64 result; 213 int status; 214 215 if (!capable(CAP_SYS_ADMIN)) 216 return -EACCES; 217 if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 218 return -EFAULT; 219 if (cmd.flags) 220 return -EINVAL; 221 if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid)) 222 return -EINVAL; 223 224 memset(&c, 0, sizeof(c)); 225 c.common.opcode = cmd.opcode; 226 c.common.flags = cmd.flags; 227 c.common.nsid = cpu_to_le32(cmd.nsid); 228 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 229 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 230 c.common.cdw10 = cpu_to_le32(cmd.cdw10); 231 c.common.cdw11 = cpu_to_le32(cmd.cdw11); 232 c.common.cdw12 = cpu_to_le32(cmd.cdw12); 233 c.common.cdw13 = cpu_to_le32(cmd.cdw13); 234 c.common.cdw14 = cpu_to_le32(cmd.cdw14); 235 c.common.cdw15 = cpu_to_le32(cmd.cdw15); 236 237 if (cmd.timeout_ms) 238 timeout = msecs_to_jiffies(cmd.timeout_ms); 239 240 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, 241 nvme_to_user_ptr(cmd.addr), cmd.data_len, 242 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len, 243 0, &result, timeout, false); 244 245 if (status >= 0) { 246 if (put_user(result, &ucmd->result)) 247 return -EFAULT; 248 } 249 250 return status; 251 } 252 253 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 254 struct nvme_passthru_cmd64 __user *ucmd, bool vec) 255 { 256 struct nvme_passthru_cmd64 cmd; 257 struct nvme_command c; 258 unsigned timeout = 0; 259 int status; 260 261 if (!capable(CAP_SYS_ADMIN)) 262 return -EACCES; 263 if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 264 return -EFAULT; 265 if (cmd.flags) 266 return -EINVAL; 267 if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid)) 268 return -EINVAL; 269 270 memset(&c, 0, sizeof(c)); 271 c.common.opcode = cmd.opcode; 272 c.common.flags = cmd.flags; 273 c.common.nsid = cpu_to_le32(cmd.nsid); 274 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 275 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 276 c.common.cdw10 = cpu_to_le32(cmd.cdw10); 277 c.common.cdw11 = cpu_to_le32(cmd.cdw11); 278 c.common.cdw12 = cpu_to_le32(cmd.cdw12); 279 c.common.cdw13 = cpu_to_le32(cmd.cdw13); 280 c.common.cdw14 = cpu_to_le32(cmd.cdw14); 281 c.common.cdw15 = cpu_to_le32(cmd.cdw15); 282 283 if (cmd.timeout_ms) 284 timeout = msecs_to_jiffies(cmd.timeout_ms); 285 286 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, 287 nvme_to_user_ptr(cmd.addr), cmd.data_len, 288 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len, 289 0, &cmd.result, timeout, vec); 290 291 if (status >= 0) { 292 if (put_user(cmd.result, &ucmd->result)) 293 return -EFAULT; 294 } 295 296 return status; 297 } 298 299 static bool is_ctrl_ioctl(unsigned int cmd) 300 { 301 if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD) 302 return true; 303 if (is_sed_ioctl(cmd)) 304 return true; 305 return false; 306 } 307 308 static int nvme_ctrl_ioctl(struct nvme_ctrl *ctrl, unsigned int cmd, 309 void __user *argp) 310 { 311 switch (cmd) { 312 case NVME_IOCTL_ADMIN_CMD: 313 return nvme_user_cmd(ctrl, NULL, argp); 314 case NVME_IOCTL_ADMIN64_CMD: 315 return nvme_user_cmd64(ctrl, NULL, argp, false); 316 default: 317 return sed_ioctl(ctrl->opal_dev, cmd, argp); 318 } 319 } 320 321 #ifdef COMPAT_FOR_U64_ALIGNMENT 322 struct nvme_user_io32 { 323 __u8 opcode; 324 __u8 flags; 325 __u16 control; 326 __u16 nblocks; 327 __u16 rsvd; 328 __u64 metadata; 329 __u64 addr; 330 __u64 slba; 331 __u32 dsmgmt; 332 __u32 reftag; 333 __u16 apptag; 334 __u16 appmask; 335 } __attribute__((__packed__)); 336 #define NVME_IOCTL_SUBMIT_IO32 _IOW('N', 0x42, struct nvme_user_io32) 337 #endif /* COMPAT_FOR_U64_ALIGNMENT */ 338 339 static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd, 340 void __user *argp) 341 { 342 switch (cmd) { 343 case NVME_IOCTL_ID: 344 force_successful_syscall_return(); 345 return ns->head->ns_id; 346 case NVME_IOCTL_IO_CMD: 347 return nvme_user_cmd(ns->ctrl, ns, argp); 348 /* 349 * struct nvme_user_io can have different padding on some 32-bit ABIs. 350 * Just accept the compat version as all fields that are used are the 351 * same size and at the same offset. 352 */ 353 #ifdef COMPAT_FOR_U64_ALIGNMENT 354 case NVME_IOCTL_SUBMIT_IO32: 355 #endif 356 case NVME_IOCTL_SUBMIT_IO: 357 return nvme_submit_io(ns, argp); 358 case NVME_IOCTL_IO64_CMD: 359 return nvme_user_cmd64(ns->ctrl, ns, argp, false); 360 case NVME_IOCTL_IO64_CMD_VEC: 361 return nvme_user_cmd64(ns->ctrl, ns, argp, true); 362 default: 363 return -ENOTTY; 364 } 365 } 366 367 static int __nvme_ioctl(struct nvme_ns *ns, unsigned int cmd, void __user *arg) 368 { 369 if (is_ctrl_ioctl(cmd)) 370 return nvme_ctrl_ioctl(ns->ctrl, cmd, arg); 371 return nvme_ns_ioctl(ns, cmd, arg); 372 } 373 374 int nvme_ioctl(struct block_device *bdev, fmode_t mode, 375 unsigned int cmd, unsigned long arg) 376 { 377 struct nvme_ns *ns = bdev->bd_disk->private_data; 378 379 return __nvme_ioctl(ns, cmd, (void __user *)arg); 380 } 381 382 long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 383 { 384 struct nvme_ns *ns = 385 container_of(file_inode(file)->i_cdev, struct nvme_ns, cdev); 386 387 return __nvme_ioctl(ns, cmd, (void __user *)arg); 388 } 389 390 #ifdef CONFIG_NVME_MULTIPATH 391 static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd, 392 void __user *argp, struct nvme_ns_head *head, int srcu_idx) 393 __releases(&head->srcu) 394 { 395 struct nvme_ctrl *ctrl = ns->ctrl; 396 int ret; 397 398 nvme_get_ctrl(ns->ctrl); 399 srcu_read_unlock(&head->srcu, srcu_idx); 400 ret = nvme_ctrl_ioctl(ns->ctrl, cmd, argp); 401 402 nvme_put_ctrl(ctrl); 403 return ret; 404 } 405 406 int nvme_ns_head_ioctl(struct block_device *bdev, fmode_t mode, 407 unsigned int cmd, unsigned long arg) 408 { 409 struct nvme_ns_head *head = bdev->bd_disk->private_data; 410 void __user *argp = (void __user *)arg; 411 struct nvme_ns *ns; 412 int srcu_idx, ret = -EWOULDBLOCK; 413 414 srcu_idx = srcu_read_lock(&head->srcu); 415 ns = nvme_find_path(head); 416 if (!ns) 417 goto out_unlock; 418 419 /* 420 * Handle ioctls that apply to the controller instead of the namespace 421 * seperately and drop the ns SRCU reference early. This avoids a 422 * deadlock when deleting namespaces using the passthrough interface. 423 */ 424 if (is_ctrl_ioctl(cmd)) 425 return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx); 426 427 ret = nvme_ns_ioctl(ns, cmd, argp); 428 out_unlock: 429 srcu_read_unlock(&head->srcu, srcu_idx); 430 return ret; 431 } 432 433 long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, 434 unsigned long arg) 435 { 436 struct cdev *cdev = file_inode(file)->i_cdev; 437 struct nvme_ns_head *head = 438 container_of(cdev, struct nvme_ns_head, cdev); 439 void __user *argp = (void __user *)arg; 440 struct nvme_ns *ns; 441 int srcu_idx, ret = -EWOULDBLOCK; 442 443 srcu_idx = srcu_read_lock(&head->srcu); 444 ns = nvme_find_path(head); 445 if (!ns) 446 goto out_unlock; 447 448 if (is_ctrl_ioctl(cmd)) 449 return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx); 450 451 ret = nvme_ns_ioctl(ns, cmd, argp); 452 out_unlock: 453 srcu_read_unlock(&head->srcu, srcu_idx); 454 return ret; 455 } 456 #endif /* CONFIG_NVME_MULTIPATH */ 457 458 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp) 459 { 460 struct nvme_ns *ns; 461 int ret; 462 463 down_read(&ctrl->namespaces_rwsem); 464 if (list_empty(&ctrl->namespaces)) { 465 ret = -ENOTTY; 466 goto out_unlock; 467 } 468 469 ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list); 470 if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) { 471 dev_warn(ctrl->device, 472 "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n"); 473 ret = -EINVAL; 474 goto out_unlock; 475 } 476 477 dev_warn(ctrl->device, 478 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n"); 479 kref_get(&ns->kref); 480 up_read(&ctrl->namespaces_rwsem); 481 482 ret = nvme_user_cmd(ctrl, ns, argp); 483 nvme_put_ns(ns); 484 return ret; 485 486 out_unlock: 487 up_read(&ctrl->namespaces_rwsem); 488 return ret; 489 } 490 491 long nvme_dev_ioctl(struct file *file, unsigned int cmd, 492 unsigned long arg) 493 { 494 struct nvme_ctrl *ctrl = file->private_data; 495 void __user *argp = (void __user *)arg; 496 497 switch (cmd) { 498 case NVME_IOCTL_ADMIN_CMD: 499 return nvme_user_cmd(ctrl, NULL, argp); 500 case NVME_IOCTL_ADMIN64_CMD: 501 return nvme_user_cmd64(ctrl, NULL, argp, false); 502 case NVME_IOCTL_IO_CMD: 503 return nvme_dev_user_cmd(ctrl, argp); 504 case NVME_IOCTL_RESET: 505 dev_warn(ctrl->device, "resetting controller\n"); 506 return nvme_reset_ctrl_sync(ctrl); 507 case NVME_IOCTL_SUBSYS_RESET: 508 return nvme_reset_subsystem(ctrl); 509 case NVME_IOCTL_RESCAN: 510 nvme_queue_scan(ctrl); 511 return 0; 512 default: 513 return -ENOTTY; 514 } 515 } 516