1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2011-2014, Intel Corporation. 4 * Copyright (c) 2017-2021 Christoph Hellwig. 5 */ 6 #include <linux/ptrace.h> /* for force_successful_syscall_return */ 7 #include <linux/nvme_ioctl.h> 8 #include "nvme.h" 9 10 /* 11 * Convert integer values from ioctl structures to user pointers, silently 12 * ignoring the upper bits in the compat case to match behaviour of 32-bit 13 * kernels. 14 */ 15 static void __user *nvme_to_user_ptr(uintptr_t ptrval) 16 { 17 if (in_compat_syscall()) 18 ptrval = (compat_uptr_t)ptrval; 19 return (void __user *)ptrval; 20 } 21 22 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf, 23 unsigned len, u32 seed, bool write) 24 { 25 struct bio_integrity_payload *bip; 26 int ret = -ENOMEM; 27 void *buf; 28 29 buf = kmalloc(len, GFP_KERNEL); 30 if (!buf) 31 goto out; 32 33 ret = -EFAULT; 34 if (write && copy_from_user(buf, ubuf, len)) 35 goto out_free_meta; 36 37 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1); 38 if (IS_ERR(bip)) { 39 ret = PTR_ERR(bip); 40 goto out_free_meta; 41 } 42 43 bip->bip_iter.bi_size = len; 44 bip->bip_iter.bi_sector = seed; 45 ret = bio_integrity_add_page(bio, virt_to_page(buf), len, 46 offset_in_page(buf)); 47 if (ret == len) 48 return buf; 49 ret = -ENOMEM; 50 out_free_meta: 51 kfree(buf); 52 out: 53 return ERR_PTR(ret); 54 } 55 56 static int nvme_submit_user_cmd(struct request_queue *q, 57 struct nvme_command *cmd, void __user *ubuffer, 58 unsigned bufflen, void __user *meta_buffer, unsigned meta_len, 59 u32 meta_seed, u64 *result, unsigned timeout) 60 { 61 bool write = nvme_is_write(cmd); 62 struct nvme_ns *ns = q->queuedata; 63 struct block_device *bdev = ns ? ns->disk->part0 : NULL; 64 struct request *req; 65 struct bio *bio = NULL; 66 void *meta = NULL; 67 int ret; 68 69 req = nvme_alloc_request(q, cmd, 0); 70 if (IS_ERR(req)) 71 return PTR_ERR(req); 72 73 if (timeout) 74 req->timeout = timeout; 75 nvme_req(req)->flags |= NVME_REQ_USERCMD; 76 77 if (ubuffer && bufflen) { 78 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen, 79 GFP_KERNEL); 80 if (ret) 81 goto out; 82 bio = req->bio; 83 if (bdev) 84 bio_set_dev(bio, bdev); 85 if (bdev && meta_buffer && meta_len) { 86 meta = nvme_add_user_metadata(bio, meta_buffer, meta_len, 87 meta_seed, write); 88 if (IS_ERR(meta)) { 89 ret = PTR_ERR(meta); 90 goto out_unmap; 91 } 92 req->cmd_flags |= REQ_INTEGRITY; 93 } 94 } 95 96 nvme_execute_passthru_rq(req); 97 if (nvme_req(req)->flags & NVME_REQ_CANCELLED) 98 ret = -EINTR; 99 else 100 ret = nvme_req(req)->status; 101 if (result) 102 *result = le64_to_cpu(nvme_req(req)->result.u64); 103 if (meta && !ret && !write) { 104 if (copy_to_user(meta_buffer, meta, meta_len)) 105 ret = -EFAULT; 106 } 107 kfree(meta); 108 out_unmap: 109 if (bio) 110 blk_rq_unmap_user(bio); 111 out: 112 blk_mq_free_request(req); 113 return ret; 114 } 115 116 117 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) 118 { 119 struct nvme_user_io io; 120 struct nvme_command c; 121 unsigned length, meta_len; 122 void __user *metadata; 123 124 if (copy_from_user(&io, uio, sizeof(io))) 125 return -EFAULT; 126 if (io.flags) 127 return -EINVAL; 128 129 switch (io.opcode) { 130 case nvme_cmd_write: 131 case nvme_cmd_read: 132 case nvme_cmd_compare: 133 break; 134 default: 135 return -EINVAL; 136 } 137 138 length = (io.nblocks + 1) << ns->lba_shift; 139 140 if ((io.control & NVME_RW_PRINFO_PRACT) && 141 ns->ms == sizeof(struct t10_pi_tuple)) { 142 /* 143 * Protection information is stripped/inserted by the 144 * controller. 145 */ 146 if (nvme_to_user_ptr(io.metadata)) 147 return -EINVAL; 148 meta_len = 0; 149 metadata = NULL; 150 } else { 151 meta_len = (io.nblocks + 1) * ns->ms; 152 metadata = nvme_to_user_ptr(io.metadata); 153 } 154 155 if (ns->features & NVME_NS_EXT_LBAS) { 156 length += meta_len; 157 meta_len = 0; 158 } else if (meta_len) { 159 if ((io.metadata & 3) || !io.metadata) 160 return -EINVAL; 161 } 162 163 memset(&c, 0, sizeof(c)); 164 c.rw.opcode = io.opcode; 165 c.rw.flags = io.flags; 166 c.rw.nsid = cpu_to_le32(ns->head->ns_id); 167 c.rw.slba = cpu_to_le64(io.slba); 168 c.rw.length = cpu_to_le16(io.nblocks); 169 c.rw.control = cpu_to_le16(io.control); 170 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt); 171 c.rw.reftag = cpu_to_le32(io.reftag); 172 c.rw.apptag = cpu_to_le16(io.apptag); 173 c.rw.appmask = cpu_to_le16(io.appmask); 174 175 return nvme_submit_user_cmd(ns->queue, &c, 176 nvme_to_user_ptr(io.addr), length, 177 metadata, meta_len, lower_32_bits(io.slba), NULL, 0); 178 } 179 180 static bool nvme_validate_passthru_nsid(struct nvme_ctrl *ctrl, 181 struct nvme_ns *ns, __u32 nsid) 182 { 183 if (ns && nsid != ns->head->ns_id) { 184 dev_err(ctrl->device, 185 "%s: nsid (%u) in cmd does not match nsid (%u)" 186 "of namespace\n", 187 current->comm, nsid, ns->head->ns_id); 188 return false; 189 } 190 191 return true; 192 } 193 194 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 195 struct nvme_passthru_cmd __user *ucmd) 196 { 197 struct nvme_passthru_cmd cmd; 198 struct nvme_command c; 199 unsigned timeout = 0; 200 u64 result; 201 int status; 202 203 if (!capable(CAP_SYS_ADMIN)) 204 return -EACCES; 205 if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 206 return -EFAULT; 207 if (cmd.flags) 208 return -EINVAL; 209 if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid)) 210 return -EINVAL; 211 212 memset(&c, 0, sizeof(c)); 213 c.common.opcode = cmd.opcode; 214 c.common.flags = cmd.flags; 215 c.common.nsid = cpu_to_le32(cmd.nsid); 216 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 217 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 218 c.common.cdw10 = cpu_to_le32(cmd.cdw10); 219 c.common.cdw11 = cpu_to_le32(cmd.cdw11); 220 c.common.cdw12 = cpu_to_le32(cmd.cdw12); 221 c.common.cdw13 = cpu_to_le32(cmd.cdw13); 222 c.common.cdw14 = cpu_to_le32(cmd.cdw14); 223 c.common.cdw15 = cpu_to_le32(cmd.cdw15); 224 225 if (cmd.timeout_ms) 226 timeout = msecs_to_jiffies(cmd.timeout_ms); 227 228 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, 229 nvme_to_user_ptr(cmd.addr), cmd.data_len, 230 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len, 231 0, &result, timeout); 232 233 if (status >= 0) { 234 if (put_user(result, &ucmd->result)) 235 return -EFAULT; 236 } 237 238 return status; 239 } 240 241 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, 242 struct nvme_passthru_cmd64 __user *ucmd) 243 { 244 struct nvme_passthru_cmd64 cmd; 245 struct nvme_command c; 246 unsigned timeout = 0; 247 int status; 248 249 if (!capable(CAP_SYS_ADMIN)) 250 return -EACCES; 251 if (copy_from_user(&cmd, ucmd, sizeof(cmd))) 252 return -EFAULT; 253 if (cmd.flags) 254 return -EINVAL; 255 if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid)) 256 return -EINVAL; 257 258 memset(&c, 0, sizeof(c)); 259 c.common.opcode = cmd.opcode; 260 c.common.flags = cmd.flags; 261 c.common.nsid = cpu_to_le32(cmd.nsid); 262 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2); 263 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3); 264 c.common.cdw10 = cpu_to_le32(cmd.cdw10); 265 c.common.cdw11 = cpu_to_le32(cmd.cdw11); 266 c.common.cdw12 = cpu_to_le32(cmd.cdw12); 267 c.common.cdw13 = cpu_to_le32(cmd.cdw13); 268 c.common.cdw14 = cpu_to_le32(cmd.cdw14); 269 c.common.cdw15 = cpu_to_le32(cmd.cdw15); 270 271 if (cmd.timeout_ms) 272 timeout = msecs_to_jiffies(cmd.timeout_ms); 273 274 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, 275 nvme_to_user_ptr(cmd.addr), cmd.data_len, 276 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len, 277 0, &cmd.result, timeout); 278 279 if (status >= 0) { 280 if (put_user(cmd.result, &ucmd->result)) 281 return -EFAULT; 282 } 283 284 return status; 285 } 286 287 static bool is_ctrl_ioctl(unsigned int cmd) 288 { 289 if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD) 290 return true; 291 if (is_sed_ioctl(cmd)) 292 return true; 293 return false; 294 } 295 296 static int nvme_ctrl_ioctl(struct nvme_ctrl *ctrl, unsigned int cmd, 297 void __user *argp) 298 { 299 switch (cmd) { 300 case NVME_IOCTL_ADMIN_CMD: 301 return nvme_user_cmd(ctrl, NULL, argp); 302 case NVME_IOCTL_ADMIN64_CMD: 303 return nvme_user_cmd64(ctrl, NULL, argp); 304 default: 305 return sed_ioctl(ctrl->opal_dev, cmd, argp); 306 } 307 } 308 309 #ifdef COMPAT_FOR_U64_ALIGNMENT 310 struct nvme_user_io32 { 311 __u8 opcode; 312 __u8 flags; 313 __u16 control; 314 __u16 nblocks; 315 __u16 rsvd; 316 __u64 metadata; 317 __u64 addr; 318 __u64 slba; 319 __u32 dsmgmt; 320 __u32 reftag; 321 __u16 apptag; 322 __u16 appmask; 323 } __attribute__((__packed__)); 324 #define NVME_IOCTL_SUBMIT_IO32 _IOW('N', 0x42, struct nvme_user_io32) 325 #endif /* COMPAT_FOR_U64_ALIGNMENT */ 326 327 static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd, 328 void __user *argp) 329 { 330 switch (cmd) { 331 case NVME_IOCTL_ID: 332 force_successful_syscall_return(); 333 return ns->head->ns_id; 334 case NVME_IOCTL_IO_CMD: 335 return nvme_user_cmd(ns->ctrl, ns, argp); 336 /* 337 * struct nvme_user_io can have different padding on some 32-bit ABIs. 338 * Just accept the compat version as all fields that are used are the 339 * same size and at the same offset. 340 */ 341 #ifdef COMPAT_FOR_U64_ALIGNMENT 342 case NVME_IOCTL_SUBMIT_IO32: 343 #endif 344 case NVME_IOCTL_SUBMIT_IO: 345 return nvme_submit_io(ns, argp); 346 case NVME_IOCTL_IO64_CMD: 347 return nvme_user_cmd64(ns->ctrl, ns, argp); 348 default: 349 if (!ns->ndev) 350 return -ENOTTY; 351 return nvme_nvm_ioctl(ns, cmd, argp); 352 } 353 } 354 355 static int __nvme_ioctl(struct nvme_ns *ns, unsigned int cmd, void __user *arg) 356 { 357 if (is_ctrl_ioctl(cmd)) 358 return nvme_ctrl_ioctl(ns->ctrl, cmd, arg); 359 return nvme_ns_ioctl(ns, cmd, arg); 360 } 361 362 int nvme_ioctl(struct block_device *bdev, fmode_t mode, 363 unsigned int cmd, unsigned long arg) 364 { 365 struct nvme_ns *ns = bdev->bd_disk->private_data; 366 367 return __nvme_ioctl(ns, cmd, (void __user *)arg); 368 } 369 370 long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 371 { 372 struct nvme_ns *ns = 373 container_of(file_inode(file)->i_cdev, struct nvme_ns, cdev); 374 375 return __nvme_ioctl(ns, cmd, (void __user *)arg); 376 } 377 378 #ifdef CONFIG_NVME_MULTIPATH 379 static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd, 380 void __user *argp, struct nvme_ns_head *head, int srcu_idx) 381 __releases(&head->srcu) 382 { 383 struct nvme_ctrl *ctrl = ns->ctrl; 384 int ret; 385 386 nvme_get_ctrl(ns->ctrl); 387 srcu_read_unlock(&head->srcu, srcu_idx); 388 ret = nvme_ctrl_ioctl(ns->ctrl, cmd, argp); 389 390 nvme_put_ctrl(ctrl); 391 return ret; 392 } 393 394 int nvme_ns_head_ioctl(struct block_device *bdev, fmode_t mode, 395 unsigned int cmd, unsigned long arg) 396 { 397 struct nvme_ns_head *head = bdev->bd_disk->private_data; 398 void __user *argp = (void __user *)arg; 399 struct nvme_ns *ns; 400 int srcu_idx, ret = -EWOULDBLOCK; 401 402 srcu_idx = srcu_read_lock(&head->srcu); 403 ns = nvme_find_path(head); 404 if (!ns) 405 goto out_unlock; 406 407 /* 408 * Handle ioctls that apply to the controller instead of the namespace 409 * seperately and drop the ns SRCU reference early. This avoids a 410 * deadlock when deleting namespaces using the passthrough interface. 411 */ 412 if (is_ctrl_ioctl(cmd)) 413 return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx); 414 415 ret = nvme_ns_ioctl(ns, cmd, argp); 416 out_unlock: 417 srcu_read_unlock(&head->srcu, srcu_idx); 418 return ret; 419 } 420 421 long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, 422 unsigned long arg) 423 { 424 struct cdev *cdev = file_inode(file)->i_cdev; 425 struct nvme_ns_head *head = 426 container_of(cdev, struct nvme_ns_head, cdev); 427 void __user *argp = (void __user *)arg; 428 struct nvme_ns *ns; 429 int srcu_idx, ret = -EWOULDBLOCK; 430 431 srcu_idx = srcu_read_lock(&head->srcu); 432 ns = nvme_find_path(head); 433 if (!ns) 434 goto out_unlock; 435 436 if (is_ctrl_ioctl(cmd)) 437 return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx); 438 439 ret = nvme_ns_ioctl(ns, cmd, argp); 440 out_unlock: 441 srcu_read_unlock(&head->srcu, srcu_idx); 442 return ret; 443 } 444 #endif /* CONFIG_NVME_MULTIPATH */ 445 446 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp) 447 { 448 struct nvme_ns *ns; 449 int ret; 450 451 down_read(&ctrl->namespaces_rwsem); 452 if (list_empty(&ctrl->namespaces)) { 453 ret = -ENOTTY; 454 goto out_unlock; 455 } 456 457 ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list); 458 if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) { 459 dev_warn(ctrl->device, 460 "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n"); 461 ret = -EINVAL; 462 goto out_unlock; 463 } 464 465 dev_warn(ctrl->device, 466 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n"); 467 kref_get(&ns->kref); 468 up_read(&ctrl->namespaces_rwsem); 469 470 ret = nvme_user_cmd(ctrl, ns, argp); 471 nvme_put_ns(ns); 472 return ret; 473 474 out_unlock: 475 up_read(&ctrl->namespaces_rwsem); 476 return ret; 477 } 478 479 long nvme_dev_ioctl(struct file *file, unsigned int cmd, 480 unsigned long arg) 481 { 482 struct nvme_ctrl *ctrl = file->private_data; 483 void __user *argp = (void __user *)arg; 484 485 switch (cmd) { 486 case NVME_IOCTL_ADMIN_CMD: 487 return nvme_user_cmd(ctrl, NULL, argp); 488 case NVME_IOCTL_ADMIN64_CMD: 489 return nvme_user_cmd64(ctrl, NULL, argp); 490 case NVME_IOCTL_IO_CMD: 491 return nvme_dev_user_cmd(ctrl, argp); 492 case NVME_IOCTL_RESET: 493 dev_warn(ctrl->device, "resetting controller\n"); 494 return nvme_reset_ctrl_sync(ctrl); 495 case NVME_IOCTL_SUBSYS_RESET: 496 return nvme_reset_subsystem(ctrl); 497 case NVME_IOCTL_RESCAN: 498 nvme_queue_scan(ctrl); 499 return 0; 500 default: 501 return -ENOTTY; 502 } 503 } 504